2024-11-19 15:13:33,813 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 15:13:33,830 main DEBUG Took 0.014445 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 15:13:33,830 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 15:13:33,831 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 15:13:33,832 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 15:13:33,834 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,844 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 15:13:33,857 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,859 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,860 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,860 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,861 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,861 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,863 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,863 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,864 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,864 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,865 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,866 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,866 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,867 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,867 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,868 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,869 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,869 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,870 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,870 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,871 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,871 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,872 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,872 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 15:13:33,873 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,873 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 15:13:33,875 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 15:13:33,877 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 15:13:33,879 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 15:13:33,880 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 15:13:33,881 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 15:13:33,882 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 15:13:33,892 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 15:13:33,895 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 15:13:33,897 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 15:13:33,897 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 15:13:33,898 main DEBUG createAppenders(={Console}) 2024-11-19 15:13:33,898 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-19 15:13:33,899 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 15:13:33,899 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-19 15:13:33,899 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 15:13:33,900 main DEBUG OutputStream closed 2024-11-19 15:13:33,900 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 15:13:33,900 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 15:13:33,901 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-19 15:13:33,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 15:13:33,980 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 15:13:33,982 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 15:13:33,983 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 15:13:33,983 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 15:13:33,984 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 15:13:33,984 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 15:13:33,985 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 15:13:33,985 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 15:13:33,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 15:13:33,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 15:13:33,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 15:13:33,987 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 15:13:33,987 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 15:13:33,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 15:13:33,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 15:13:33,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 15:13:33,989 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 15:13:33,991 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 15:13:33,992 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-19 15:13:33,992 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 15:13:33,993 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-19T15:13:34,248 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8 2024-11-19 15:13:34,250 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 15:13:34,251 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T15:13:34,260 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-19T15:13:34,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T15:13:34,298 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/cluster_26e20745-1875-184e-c7e5-36298de089d9, deleteOnExit=true 2024-11-19T15:13:34,298 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T15:13:34,300 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/test.cache.data in system properties and HBase conf 2024-11-19T15:13:34,300 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T15:13:34,301 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/hadoop.log.dir in system properties and HBase conf 2024-11-19T15:13:34,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T15:13:34,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T15:13:34,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T15:13:34,399 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T15:13:34,494 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T15:13:34,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T15:13:34,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T15:13:34,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T15:13:34,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T15:13:34,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T15:13:34,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T15:13:34,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T15:13:34,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T15:13:34,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T15:13:34,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/nfs.dump.dir in system properties and HBase conf 2024-11-19T15:13:34,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/java.io.tmpdir in system properties and HBase conf 2024-11-19T15:13:34,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T15:13:34,505 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T15:13:34,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T15:13:35,311 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T15:13:35,385 INFO [Time-limited test {}] log.Log(170): Logging initialized @2300ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T15:13:35,461 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T15:13:35,520 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T15:13:35,540 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T15:13:35,541 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T15:13:35,542 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T15:13:35,554 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T15:13:35,557 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/hadoop.log.dir/,AVAILABLE} 2024-11-19T15:13:35,558 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T15:13:35,746 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/java.io.tmpdir/jetty-localhost-39297-hadoop-hdfs-3_4_1-tests_jar-_-any-16619224575086920549/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T15:13:35,753 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:39297} 2024-11-19T15:13:35,753 INFO [Time-limited test {}] server.Server(415): Started @2669ms 2024-11-19T15:13:36,145 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T15:13:36,152 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T15:13:36,153 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T15:13:36,153 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T15:13:36,153 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T15:13:36,154 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/hadoop.log.dir/,AVAILABLE} 2024-11-19T15:13:36,155 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T15:13:36,275 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/java.io.tmpdir/jetty-localhost-40851-hadoop-hdfs-3_4_1-tests_jar-_-any-18232734826785832003/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T15:13:36,276 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:40851} 2024-11-19T15:13:36,276 INFO [Time-limited test {}] server.Server(415): Started @3193ms 2024-11-19T15:13:36,332 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T15:13:36,797 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/cluster_26e20745-1875-184e-c7e5-36298de089d9/data/data1/current/BP-618166336-172.17.0.2-1732029215069/current, will proceed with Du for space computation calculation, 2024-11-19T15:13:36,797 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/cluster_26e20745-1875-184e-c7e5-36298de089d9/data/data2/current/BP-618166336-172.17.0.2-1732029215069/current, will proceed with Du for space computation calculation, 2024-11-19T15:13:36,860 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T15:13:36,922 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x99e9d2cde765685d with lease ID 0xde64587c33dcb6cc: Processing first storage report for DS-69d0b32a-66fd-45c5-a947-7c5abc4db381 from datanode DatanodeRegistration(127.0.0.1:42567, datanodeUuid=99eab563-9309-42c7-bee5-e233e87de811, infoPort=39469, infoSecurePort=0, ipcPort=44095, storageInfo=lv=-57;cid=testClusterID;nsid=1412924323;c=1732029215069) 2024-11-19T15:13:36,925 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x99e9d2cde765685d with lease ID 0xde64587c33dcb6cc: from storage DS-69d0b32a-66fd-45c5-a947-7c5abc4db381 node DatanodeRegistration(127.0.0.1:42567, datanodeUuid=99eab563-9309-42c7-bee5-e233e87de811, infoPort=39469, infoSecurePort=0, ipcPort=44095, storageInfo=lv=-57;cid=testClusterID;nsid=1412924323;c=1732029215069), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-19T15:13:36,925 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x99e9d2cde765685d with lease ID 0xde64587c33dcb6cc: Processing first storage report for DS-8742c5da-79fb-4818-a18b-567d45f6c722 from datanode DatanodeRegistration(127.0.0.1:42567, datanodeUuid=99eab563-9309-42c7-bee5-e233e87de811, infoPort=39469, infoSecurePort=0, ipcPort=44095, storageInfo=lv=-57;cid=testClusterID;nsid=1412924323;c=1732029215069) 2024-11-19T15:13:36,925 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x99e9d2cde765685d with lease ID 0xde64587c33dcb6cc: from storage DS-8742c5da-79fb-4818-a18b-567d45f6c722 node DatanodeRegistration(127.0.0.1:42567, datanodeUuid=99eab563-9309-42c7-bee5-e233e87de811, infoPort=39469, infoSecurePort=0, ipcPort=44095, storageInfo=lv=-57;cid=testClusterID;nsid=1412924323;c=1732029215069), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T15:13:36,982 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8 2024-11-19T15:13:37,058 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/cluster_26e20745-1875-184e-c7e5-36298de089d9/zookeeper_0, clientPort=60923, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/cluster_26e20745-1875-184e-c7e5-36298de089d9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/cluster_26e20745-1875-184e-c7e5-36298de089d9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T15:13:37,069 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60923 2024-11-19T15:13:37,078 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T15:13:37,081 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T15:13:37,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741825_1001 (size=7) 2024-11-19T15:13:37,741 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8 with version=8 2024-11-19T15:13:37,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/hbase-staging 2024-11-19T15:13:37,836 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T15:13:38,116 INFO [Time-limited test {}] client.ConnectionUtils(128): master/50570b5b74e7:0 server-side Connection retries=45 2024-11-19T15:13:38,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T15:13:38,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T15:13:38,132 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T15:13:38,132 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T15:13:38,132 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T15:13:38,271 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T15:13:38,333 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T15:13:38,346 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T15:13:38,352 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T15:13:38,381 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3074 (auto-detected) 2024-11-19T15:13:38,382 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T15:13:38,401 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39501 2024-11-19T15:13:38,423 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39501 connecting to ZooKeeper ensemble=127.0.0.1:60923 2024-11-19T15:13:38,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:395010x0, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T15:13:38,459 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39501-0x10130ca639e0000 connected 2024-11-19T15:13:38,491 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T15:13:38,493 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T15:13:38,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T15:13:38,509 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8, hbase.cluster.distributed=false 2024-11-19T15:13:38,536 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T15:13:38,543 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39501 2024-11-19T15:13:38,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39501 2024-11-19T15:13:38,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39501 2024-11-19T15:13:38,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39501 2024-11-19T15:13:38,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39501 2024-11-19T15:13:38,697 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/50570b5b74e7:0 server-side Connection retries=45 2024-11-19T15:13:38,700 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T15:13:38,700 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T15:13:38,700 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T15:13:38,701 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T15:13:38,701 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T15:13:38,705 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T15:13:38,708 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T15:13:38,709 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46557 2024-11-19T15:13:38,710 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46557 connecting to ZooKeeper ensemble=127.0.0.1:60923 2024-11-19T15:13:38,712 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T15:13:38,715 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T15:13:38,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:465570x0, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T15:13:38,722 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:465570x0, quorum=127.0.0.1:60923, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T15:13:38,722 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46557-0x10130ca639e0001 connected 2024-11-19T15:13:38,727 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T15:13:38,737 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T15:13:38,740 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T15:13:38,746 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T15:13:38,747 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46557 2024-11-19T15:13:38,748 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46557 2024-11-19T15:13:38,748 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46557 2024-11-19T15:13:38,750 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46557 2024-11-19T15:13:38,751 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46557 2024-11-19T15:13:38,771 DEBUG [M:0;50570b5b74e7:39501 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;50570b5b74e7:39501 2024-11-19T15:13:38,772 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/50570b5b74e7,39501,1732029217890 2024-11-19T15:13:38,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T15:13:38,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T15:13:38,780 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/50570b5b74e7,39501,1732029217890 2024-11-19T15:13:38,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T15:13:38,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:13:38,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:13:38,801 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T15:13:38,802 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/50570b5b74e7,39501,1732029217890 from backup master directory 2024-11-19T15:13:38,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/50570b5b74e7,39501,1732029217890 2024-11-19T15:13:38,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T15:13:38,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T15:13:38,807 WARN [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T15:13:38,807 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=50570b5b74e7,39501,1732029217890 2024-11-19T15:13:38,810 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T15:13:38,811 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T15:13:38,876 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/hbase.id] with ID: 3dd3724b-b902-4c12-9084-cb3290376b64 2024-11-19T15:13:38,876 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/.tmp/hbase.id 2024-11-19T15:13:38,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741826_1002 (size=42) 2024-11-19T15:13:39,291 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/.tmp/hbase.id]:[hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/hbase.id] 2024-11-19T15:13:39,343 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T15:13:39,349 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T15:13:39,372 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-19T15:13:39,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:13:39,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:13:39,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741827_1003 (size=196) 2024-11-19T15:13:39,811 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T15:13:39,814 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T15:13:39,834 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:39,840 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T15:13:39,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741828_1004 (size=1189) 2024-11-19T15:13:40,308 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store 2024-11-19T15:13:40,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741829_1005 (size=34) 2024-11-19T15:13:40,341 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T15:13:40,345 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:13:40,346 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T15:13:40,346 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T15:13:40,347 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T15:13:40,349 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T15:13:40,349 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T15:13:40,349 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T15:13:40,351 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732029220346Disabling compacts and flushes for region at 1732029220346Disabling writes for close at 1732029220349 (+3 ms)Writing region close event to WAL at 1732029220349Closed at 1732029220349 2024-11-19T15:13:40,354 WARN [master/50570b5b74e7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/.initializing 2024-11-19T15:13:40,354 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/WALs/50570b5b74e7,39501,1732029217890 2024-11-19T15:13:40,370 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T15:13:40,392 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=50570b5b74e7%2C39501%2C1732029217890, suffix=, logDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/WALs/50570b5b74e7,39501,1732029217890, archiveDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/oldWALs, maxLogs=10 2024-11-19T15:13:40,429 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/WALs/50570b5b74e7,39501,1732029217890/50570b5b74e7%2C39501%2C1732029217890.1732029220398, exclude list is [], retry=0 2024-11-19T15:13:40,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42567,DS-69d0b32a-66fd-45c5-a947-7c5abc4db381,DISK] 2024-11-19T15:13:40,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T15:13:40,515 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/WALs/50570b5b74e7,39501,1732029217890/50570b5b74e7%2C39501%2C1732029217890.1732029220398 2024-11-19T15:13:40,516 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39469:39469)] 2024-11-19T15:13:40,517 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T15:13:40,518 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:13:40,522 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T15:13:40,524 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T15:13:40,580 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T15:13:40,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T15:13:40,624 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:40,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T15:13:40,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T15:13:40,635 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T15:13:40,636 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:40,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:13:40,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T15:13:40,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T15:13:40,642 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:40,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:13:40,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T15:13:40,648 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T15:13:40,649 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:40,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:13:40,651 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T15:13:40,656 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T15:13:40,658 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T15:13:40,666 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T15:13:40,667 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T15:13:40,671 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T15:13:40,675 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T15:13:40,680 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T15:13:40,682 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64930081, jitterRate=-0.03246639668941498}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T15:13:40,689 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732029220544Initializing all the Stores at 1732029220547 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732029220548 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029220549 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029220549Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029220549Cleaning up temporary data from old regions at 1732029220667 (+118 ms)Region opened successfully at 1732029220689 (+22 ms) 2024-11-19T15:13:40,691 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T15:13:40,748 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f684b27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=50570b5b74e7/172.17.0.2:0 2024-11-19T15:13:40,797 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T15:13:40,814 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T15:13:40,814 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T15:13:40,819 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T15:13:40,821 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-19T15:13:40,828 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-19T15:13:40,829 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T15:13:40,869 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T15:13:40,879 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T15:13:40,882 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T15:13:40,885 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T15:13:40,887 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T15:13:40,889 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T15:13:40,893 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T15:13:40,898 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T15:13:40,900 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T15:13:40,902 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T15:13:40,904 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T15:13:40,926 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T15:13:40,927 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T15:13:40,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T15:13:40,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T15:13:40,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:13:40,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:13:40,936 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=50570b5b74e7,39501,1732029217890, sessionid=0x10130ca639e0000, setting cluster-up flag (Was=false) 2024-11-19T15:13:40,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:13:40,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:13:40,960 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T15:13:40,962 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=50570b5b74e7,39501,1732029217890 2024-11-19T15:13:40,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:13:40,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:13:40,975 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T15:13:40,977 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=50570b5b74e7,39501,1732029217890 2024-11-19T15:13:40,986 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T15:13:41,061 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(746): ClusterId : 3dd3724b-b902-4c12-9084-cb3290376b64 2024-11-19T15:13:41,064 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T15:13:41,069 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T15:13:41,070 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T15:13:41,073 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T15:13:41,074 DEBUG [RS:0;50570b5b74e7:46557 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db5a916, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=50570b5b74e7/172.17.0.2:0 2024-11-19T15:13:41,088 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T15:13:41,092 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;50570b5b74e7:46557 2024-11-19T15:13:41,095 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T15:13:41,095 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T15:13:41,095 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T15:13:41,098 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(2659): reportForDuty to master=50570b5b74e7,39501,1732029217890 with port=46557, startcode=1732029218639 2024-11-19T15:13:41,099 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T15:13:41,105 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T15:13:41,110 DEBUG [RS:0;50570b5b74e7:46557 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T15:13:41,111 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 50570b5b74e7,39501,1732029217890 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T15:13:41,118 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/50570b5b74e7:0, corePoolSize=5, maxPoolSize=5 2024-11-19T15:13:41,118 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/50570b5b74e7:0, corePoolSize=5, maxPoolSize=5 2024-11-19T15:13:41,119 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/50570b5b74e7:0, corePoolSize=5, maxPoolSize=5 2024-11-19T15:13:41,119 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/50570b5b74e7:0, corePoolSize=5, maxPoolSize=5 2024-11-19T15:13:41,119 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/50570b5b74e7:0, corePoolSize=10, maxPoolSize=10 2024-11-19T15:13:41,119 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,119 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/50570b5b74e7:0, corePoolSize=2, maxPoolSize=2 2024-11-19T15:13:41,120 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,123 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732029251123 2024-11-19T15:13:41,124 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T15:13:41,126 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T15:13:41,127 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T15:13:41,128 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T15:13:41,131 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T15:13:41,131 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T15:13:41,131 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T15:13:41,132 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T15:13:41,132 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,135 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:41,135 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T15:13:41,137 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T15:13:41,139 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T15:13:41,139 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T15:13:41,142 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T15:13:41,142 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T15:13:41,145 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/50570b5b74e7:0:becomeActiveMaster-HFileCleaner.large.0-1732029221144,5,FailOnTimeoutGroup] 2024-11-19T15:13:41,146 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/50570b5b74e7:0:becomeActiveMaster-HFileCleaner.small.0-1732029221145,5,FailOnTimeoutGroup] 2024-11-19T15:13:41,146 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,146 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T15:13:41,148 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,149 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741831_1007 (size=1321) 2024-11-19T15:13:41,149 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56663, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T15:13:41,155 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39501 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 50570b5b74e7,46557,1732029218639 2024-11-19T15:13:41,158 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39501 {}] master.ServerManager(517): Registering regionserver=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:41,172 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8 2024-11-19T15:13:41,173 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36139 2024-11-19T15:13:41,173 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T15:13:41,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T15:13:41,179 DEBUG [RS:0;50570b5b74e7:46557 {}] zookeeper.ZKUtil(111): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/50570b5b74e7,46557,1732029218639 2024-11-19T15:13:41,179 WARN [RS:0;50570b5b74e7:46557 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T15:13:41,179 INFO [RS:0;50570b5b74e7:46557 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T15:13:41,180 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/WALs/50570b5b74e7,46557,1732029218639 2024-11-19T15:13:41,181 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [50570b5b74e7,46557,1732029218639] 2024-11-19T15:13:41,207 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T15:13:41,221 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T15:13:41,226 INFO [RS:0;50570b5b74e7:46557 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T15:13:41,226 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,227 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T15:13:41,234 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T15:13:41,235 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,235 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,235 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,236 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,236 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,236 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,236 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/50570b5b74e7:0, corePoolSize=2, maxPoolSize=2 2024-11-19T15:13:41,236 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,236 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,237 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,237 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,237 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,237 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/50570b5b74e7:0, corePoolSize=1, maxPoolSize=1 2024-11-19T15:13:41,237 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/50570b5b74e7:0, corePoolSize=3, maxPoolSize=3 2024-11-19T15:13:41,237 DEBUG [RS:0;50570b5b74e7:46557 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0, corePoolSize=3, maxPoolSize=3 2024-11-19T15:13:41,238 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,239 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,239 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,239 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,239 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,239 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ChoreService(168): Chore ScheduledChore name=50570b5b74e7,46557,1732029218639-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T15:13:41,262 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T15:13:41,264 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ChoreService(168): Chore ScheduledChore name=50570b5b74e7,46557,1732029218639-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,264 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,264 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.Replication(171): 50570b5b74e7,46557,1732029218639 started 2024-11-19T15:13:41,284 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:41,284 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1482): Serving as 50570b5b74e7,46557,1732029218639, RpcServer on 50570b5b74e7/172.17.0.2:46557, sessionid=0x10130ca639e0001 2024-11-19T15:13:41,285 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T15:13:41,285 DEBUG [RS:0;50570b5b74e7:46557 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 50570b5b74e7,46557,1732029218639 2024-11-19T15:13:41,285 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '50570b5b74e7,46557,1732029218639' 2024-11-19T15:13:41,286 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T15:13:41,287 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T15:13:41,287 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T15:13:41,287 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T15:13:41,288 DEBUG [RS:0;50570b5b74e7:46557 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 50570b5b74e7,46557,1732029218639 2024-11-19T15:13:41,288 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '50570b5b74e7,46557,1732029218639' 2024-11-19T15:13:41,288 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T15:13:41,288 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T15:13:41,289 DEBUG [RS:0;50570b5b74e7:46557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T15:13:41,289 INFO [RS:0;50570b5b74e7:46557 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T15:13:41,289 INFO [RS:0;50570b5b74e7:46557 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T15:13:41,395 INFO [RS:0;50570b5b74e7:46557 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T15:13:41,399 INFO [RS:0;50570b5b74e7:46557 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=50570b5b74e7%2C46557%2C1732029218639, suffix=, logDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/WALs/50570b5b74e7,46557,1732029218639, archiveDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/oldWALs, maxLogs=32 2024-11-19T15:13:41,419 DEBUG [RS:0;50570b5b74e7:46557 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/WALs/50570b5b74e7,46557,1732029218639/50570b5b74e7%2C46557%2C1732029218639.1732029221402, exclude list is [], retry=0 2024-11-19T15:13:41,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42567,DS-69d0b32a-66fd-45c5-a947-7c5abc4db381,DISK] 2024-11-19T15:13:41,428 INFO [RS:0;50570b5b74e7:46557 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/WALs/50570b5b74e7,46557,1732029218639/50570b5b74e7%2C46557%2C1732029218639.1732029221402 2024-11-19T15:13:41,428 DEBUG [RS:0;50570b5b74e7:46557 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39469:39469)] 2024-11-19T15:13:41,552 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T15:13:41,552 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8 2024-11-19T15:13:41,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741833_1009 (size=32) 2024-11-19T15:13:41,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:13:41,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T15:13:41,971 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T15:13:41,971 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:41,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T15:13:41,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T15:13:41,975 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T15:13:41,976 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:41,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T15:13:41,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T15:13:41,979 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T15:13:41,979 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:41,980 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T15:13:41,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T15:13:41,983 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T15:13:41,983 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:41,984 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T15:13:41,985 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T15:13:41,986 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740 2024-11-19T15:13:41,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740 2024-11-19T15:13:41,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T15:13:41,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T15:13:41,991 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T15:13:41,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T15:13:41,999 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T15:13:42,000 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69831195, jitterRate=0.040565893054008484}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T15:13:42,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732029221965Initializing all the Stores at 1732029221967 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732029221967Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732029221967Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029221967Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732029221967Cleaning up temporary data from old regions at 1732029221990 (+23 ms)Region opened successfully at 1732029222002 (+12 ms) 2024-11-19T15:13:42,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T15:13:42,003 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T15:13:42,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T15:13:42,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T15:13:42,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T15:13:42,004 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T15:13:42,004 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732029222002Disabling compacts and flushes for region at 1732029222002Disabling writes for close at 1732029222003 (+1 ms)Writing region close event to WAL at 1732029222004 (+1 ms)Closed at 1732029222004 2024-11-19T15:13:42,008 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T15:13:42,008 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T15:13:42,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T15:13:42,027 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T15:13:42,030 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T15:13:42,184 DEBUG [50570b5b74e7:39501 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T15:13:42,199 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:42,207 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 50570b5b74e7,46557,1732029218639, state=OPENING 2024-11-19T15:13:42,214 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T15:13:42,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:13:42,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:13:42,218 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T15:13:42,219 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T15:13:42,220 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T15:13:42,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:13:42,402 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T15:13:42,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57029, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T15:13:42,416 INFO [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T15:13:42,416 INFO [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T15:13:42,417 INFO [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T15:13:42,420 INFO [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=50570b5b74e7%2C46557%2C1732029218639.meta, suffix=.meta, logDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/WALs/50570b5b74e7,46557,1732029218639, archiveDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/oldWALs, maxLogs=32 2024-11-19T15:13:42,437 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/WALs/50570b5b74e7,46557,1732029218639/50570b5b74e7%2C46557%2C1732029218639.meta.1732029222422.meta, exclude list is [], retry=0 2024-11-19T15:13:42,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42567,DS-69d0b32a-66fd-45c5-a947-7c5abc4db381,DISK] 2024-11-19T15:13:42,444 INFO [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/WALs/50570b5b74e7,46557,1732029218639/50570b5b74e7%2C46557%2C1732029218639.meta.1732029222422.meta 2024-11-19T15:13:42,444 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39469:39469)] 2024-11-19T15:13:42,445 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T15:13:42,446 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T15:13:42,449 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T15:13:42,454 INFO [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T15:13:42,459 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T15:13:42,459 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:13:42,460 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T15:13:42,460 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T15:13:42,463 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T15:13:42,465 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T15:13:42,465 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:42,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T15:13:42,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T15:13:42,468 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T15:13:42,468 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:42,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T15:13:42,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T15:13:42,471 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T15:13:42,471 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:42,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T15:13:42,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T15:13:42,473 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T15:13:42,473 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:42,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T15:13:42,474 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T15:13:42,475 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740 2024-11-19T15:13:42,478 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740 2024-11-19T15:13:42,481 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T15:13:42,481 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T15:13:42,482 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T15:13:42,484 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T15:13:42,486 INFO [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70937661, jitterRate=0.057053521275520325}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T15:13:42,486 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T15:13:42,487 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732029222460Writing region info on filesystem at 1732029222460Initializing all the Stores at 1732029222462 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732029222462Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732029222463 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029222463Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732029222463Cleaning up temporary data from old regions at 1732029222481 (+18 ms)Running coprocessor post-open hooks at 1732029222486 (+5 ms)Region opened successfully at 1732029222487 (+1 ms) 2024-11-19T15:13:42,495 INFO [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732029222392 2024-11-19T15:13:42,506 DEBUG [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T15:13:42,506 INFO [RS_OPEN_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T15:13:42,508 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:42,509 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 50570b5b74e7,46557,1732029218639, state=OPEN 2024-11-19T15:13:42,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T15:13:42,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T15:13:42,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T15:13:42,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T15:13:42,518 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:42,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T15:13:42,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=50570b5b74e7,46557,1732029218639 in 296 msec 2024-11-19T15:13:42,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T15:13:42,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 510 msec 2024-11-19T15:13:42,532 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T15:13:42,532 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T15:13:42,553 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:13:42,554 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:13:42,575 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:13:42,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40973, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:13:42,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5730 sec 2024-11-19T15:13:42,603 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732029222602, completionTime=-1 2024-11-19T15:13:42,605 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T15:13:42,605 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T15:13:42,631 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-19T15:13:42,631 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732029282631 2024-11-19T15:13:42,631 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732029342631 2024-11-19T15:13:42,631 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 26 msec 2024-11-19T15:13:42,634 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=50570b5b74e7,39501,1732029217890-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:42,634 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=50570b5b74e7,39501,1732029217890-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:42,634 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=50570b5b74e7,39501,1732029217890-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:42,636 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-50570b5b74e7:39501, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:42,636 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:42,637 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:42,642 DEBUG [master/50570b5b74e7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T15:13:42,669 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.860sec 2024-11-19T15:13:42,670 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T15:13:42,671 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T15:13:42,673 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T15:13:42,673 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T15:13:42,673 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T15:13:42,674 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=50570b5b74e7,39501,1732029217890-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T15:13:42,675 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=50570b5b74e7,39501,1732029217890-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T15:13:42,684 DEBUG [master/50570b5b74e7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T15:13:42,685 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T15:13:42,686 INFO [master/50570b5b74e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=50570b5b74e7,39501,1732029217890-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T15:13:42,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:42,775 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T15:13:42,775 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T15:13:42,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:13:42,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:13:42,789 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:13:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:13:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:13:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad18757, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:13:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:13:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:13:42,798 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:13:42,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:42,801 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:13:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:13:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:13:42,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:13:42,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=50570b5b74e7,39501,1732029217890 2024-11-19T15:13:42,862 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=8400 2024-11-19T15:13:42,875 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T15:13:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 50570b5b74e7,39501,1732029217890 2024-11-19T15:13:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@216dfbe8 2024-11-19T15:13:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T15:13:42,887 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T15:13:42,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T15:13:42,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T15:13:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T15:13:42,904 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T15:13:42,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-19T15:13:42,907 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:42,909 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T15:13:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T15:13:42,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741835_1011 (size=1061) 2024-11-19T15:13:42,924 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8 2024-11-19T15:13:42,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741836_1012 (size=53) 2024-11-19T15:13:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T15:13:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T15:13:43,335 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:13:43,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9a10ee694987b867f252a8f623c69480, disabling compactions & flushes 2024-11-19T15:13:43,336 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:43,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:43,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. after waiting 0 ms 2024-11-19T15:13:43,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:43,336 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:43,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9a10ee694987b867f252a8f623c69480: Waiting for close lock at 1732029223335Disabling compacts and flushes for region at 1732029223335Disabling writes for close at 1732029223336 (+1 ms)Writing region close event to WAL at 1732029223336Closed at 1732029223336 2024-11-19T15:13:43,339 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T15:13:43,346 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732029223339"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732029223339"}]},"ts":"1732029223339"} 2024-11-19T15:13:43,353 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T15:13:43,356 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T15:13:43,360 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029223357"}]},"ts":"1732029223357"} 2024-11-19T15:13:43,367 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T15:13:43,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a10ee694987b867f252a8f623c69480, ASSIGN}] 2024-11-19T15:13:43,372 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a10ee694987b867f252a8f623c69480, ASSIGN 2024-11-19T15:13:43,374 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a10ee694987b867f252a8f623c69480, ASSIGN; state=OFFLINE, location=50570b5b74e7,46557,1732029218639; forceNewPlan=false, retain=false 2024-11-19T15:13:43,526 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9a10ee694987b867f252a8f623c69480, regionState=OPENING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:43,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a10ee694987b867f252a8f623c69480, ASSIGN because future has completed 2024-11-19T15:13:43,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:13:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T15:13:43,694 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:43,694 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} 2024-11-19T15:13:43,695 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,695 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:13:43,695 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,695 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,698 INFO [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,701 INFO [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:13:43,701 INFO [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a10ee694987b867f252a8f623c69480 columnFamilyName A 2024-11-19T15:13:43,701 DEBUG [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:43,702 INFO [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] regionserver.HStore(327): Store=9a10ee694987b867f252a8f623c69480/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:13:43,703 INFO [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,705 INFO [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:13:43,705 INFO [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a10ee694987b867f252a8f623c69480 columnFamilyName B 2024-11-19T15:13:43,705 DEBUG [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:43,706 INFO [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] regionserver.HStore(327): Store=9a10ee694987b867f252a8f623c69480/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:13:43,706 INFO [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,708 INFO [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:13:43,709 INFO [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a10ee694987b867f252a8f623c69480 columnFamilyName C 2024-11-19T15:13:43,709 DEBUG [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:13:43,710 INFO [StoreOpener-9a10ee694987b867f252a8f623c69480-1 {}] regionserver.HStore(327): Store=9a10ee694987b867f252a8f623c69480/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:13:43,711 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:43,711 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,713 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,713 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,715 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,715 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,716 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T15:13:43,719 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,722 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T15:13:43,723 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9a10ee694987b867f252a8f623c69480; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65064615, jitterRate=-0.030461683869361877}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T15:13:43,724 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:43,725 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9a10ee694987b867f252a8f623c69480: Running coprocessor pre-open hook at 1732029223696Writing region info on filesystem at 1732029223696Initializing all the Stores at 1732029223697 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029223697Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029223697Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029223697Cleaning up temporary data from old regions at 1732029223715 (+18 ms)Running coprocessor post-open hooks at 1732029223724 (+9 ms)Region opened successfully at 1732029223725 (+1 ms) 2024-11-19T15:13:43,727 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., pid=6, masterSystemTime=1732029223687 2024-11-19T15:13:43,731 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:43,731 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:43,732 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9a10ee694987b867f252a8f623c69480, regionState=OPEN, openSeqNum=2, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:43,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:13:43,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T15:13:43,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 in 207 msec 2024-11-19T15:13:43,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T15:13:43,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a10ee694987b867f252a8f623c69480, ASSIGN in 374 msec 2024-11-19T15:13:43,748 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T15:13:43,749 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029223749"}]},"ts":"1732029223749"} 2024-11-19T15:13:43,752 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T15:13:43,754 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T15:13:43,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 857 msec 2024-11-19T15:13:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T15:13:44,056 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:13:44,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:13:44,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:13:44,064 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:13:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:13:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:13:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60daaef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:13:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:13:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:13:44,067 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:13:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bebe1d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,069 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:13:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:13:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:13:44,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:13:44,076 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T15:13:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 50570b5b74e7,39501,1732029217890 2024-11-19T15:13:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c7f33c8 2024-11-19T15:13:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T15:13:44,082 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T15:13:44,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,088 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:13:44,088 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:13:44,089 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:13:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:13:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:13:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f5a6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:13:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:13:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:13:44,093 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:13:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b383f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,095 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:13:44,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:13:44,097 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:13:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:13:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:13:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175526a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:13:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:13:44,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:13:44,104 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:13:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:13:44,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:13:44,108 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:13:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:13:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:13:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:13:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:13:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:13:44,113 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:13:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,117 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:13:44,117 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:13:44,123 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:13:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:13:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:13:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf98d02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:13:44,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:13:44,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:13:44,128 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:13:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35159008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:13:44,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:13:44,133 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:13:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:13:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:13:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4965874, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:13:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:13:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:13:44,136 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:13:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54571dc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,140 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:13:44,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:13:44,141 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:13:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:13:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:13:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62fad981, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:13:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:13:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:13:44,145 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:13:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf3ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,147 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:13:44,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:13:44,148 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:13:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:13:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:13:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b90827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:13:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:13:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:13:44,152 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:13:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45584418, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,157 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:13:44,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:13:44,158 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:13:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:13:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:13:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f11d0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:13:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:13:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:13:44,162 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:13:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b92dc16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:13:44,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:13:44,166 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:13:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:13:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:13:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6602a8c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:13:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:13:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:13:44,171 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:13:44,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7933c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:13:44,180 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:13:44,180 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:13:44,181 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:13:44,181 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:13:44,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:13:44,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:13:44,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:13:44,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:13:44,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:13:44,186 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:13:44,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:13:44,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:13:44,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:13:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:13:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:13:44,188 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:13:44,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:13:44,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:13:44,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:13:44,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:13:44,195 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:13:44,201 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:13:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:13:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:13:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-19T15:13:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:13:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:13:44,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:13:44,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:13:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:13:44,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:13:44,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:13:44,214 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:13:44,220 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:13:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T15:13:44,222 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:13:44,222 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:13:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:13:44,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:13:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:13:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:13:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:13:44,226 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:13:44,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:13:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:13:44,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:13:44,232 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:13:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:13:44,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:13:44,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:13:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:13:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:44,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:13:44,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:44,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:44,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:44,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:44,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:44,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T15:13:44,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:44,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:44,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:44,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/2b534ab50dba4556987f571fe0d28a45 is 50, key is test_row_0/A:col10/1732029224291/Put/seqid=0 2024-11-19T15:13:44,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:44,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:44,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:44,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741837_1013 (size=12001) 2024-11-19T15:13:44,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/2b534ab50dba4556987f571fe0d28a45 2024-11-19T15:13:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029284452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029284465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029224440, max timestamp=1732029224466Num uniques 859; 2024-11-19T15:13:44,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:44,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029224440, max timestamp=1732029224466Num uniques 859; 2024-11-19T15:13:44,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:44,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029224440, max timestamp=1732029224471Num uniques 859; 2024-11-19T15:13:44,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T15:13:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/78a112a56064478286aa1952674b4e35 is 50, key is test_row_0/B:col10/1732029224291/Put/seqid=0 2024-11-19T15:13:44,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:44,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029284589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029284595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029284614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:44,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:44,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:44,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:44,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741838_1014 (size=12001) 2024-11-19T15:13:44,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:44,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029284656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029284665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029284726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029284736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029284737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:44,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:44,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:44,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:44,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T15:13:44,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029284885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029284886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:44,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:44,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:44,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:44,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:44,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:44,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029284945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029284946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:44,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:44,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029284955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:44,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:45,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/78a112a56064478286aa1952674b4e35 2024-11-19T15:13:45,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:45,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:45,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/722ee79694394407b104c3be58b398fd is 50, key is test_row_0/C:col10/1732029224291/Put/seqid=0 2024-11-19T15:13:45,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741839_1015 (size=12001) 2024-11-19T15:13:45,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029285195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:45,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:45,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029285205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:45,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:45,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029285255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:45,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:45,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:45,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:45,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:45,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029285265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:45,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029285280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T15:13:45,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:45,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:45,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:45,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:45,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/722ee79694394407b104c3be58b398fd 2024-11-19T15:13:45,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/2b534ab50dba4556987f571fe0d28a45 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/2b534ab50dba4556987f571fe0d28a45 2024-11-19T15:13:45,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:45,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:45,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:45,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:45,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/2b534ab50dba4556987f571fe0d28a45, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T15:13:45,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/78a112a56064478286aa1952674b4e35 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/78a112a56064478286aa1952674b4e35 2024-11-19T15:13:45,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/78a112a56064478286aa1952674b4e35, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T15:13:45,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/722ee79694394407b104c3be58b398fd as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/722ee79694394407b104c3be58b398fd 2024-11-19T15:13:45,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/722ee79694394407b104c3be58b398fd, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T15:13:45,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9a10ee694987b867f252a8f623c69480 in 1355ms, sequenceid=12, compaction requested=false 2024-11-19T15:13:45,666 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T15:13:45,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:45,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=134.18 KB heapSize=385.22 KB 2024-11-19T15:13:45,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:45,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:45,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:45,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:45,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:45,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T15:13:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T15:13:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T15:13:45,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/c33944b29bdc40b49b4c01fe9681f9ad is 50, key is test_row_0/A:col10/1732029225717/Put/seqid=0 2024-11-19T15:13:45,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741840_1016 (size=14341) 2024-11-19T15:13:45,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/c33944b29bdc40b49b4c01fe9681f9ad 2024-11-19T15:13:45,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029285838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:45,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/88fa628b233844c385103c8ee86cfbd2 is 50, key is test_row_0/B:col10/1732029225717/Put/seqid=0 2024-11-19T15:13:45,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029285847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:45,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:45,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:45,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:45,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:45,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:45,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:45,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029285855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:45,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029285857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:45,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029285863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:45,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741841_1017 (size=12001) 2024-11-19T15:13:45,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029285965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:46,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029285995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:46,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029286015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:46,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:46,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029286015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:46,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029286016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:46,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:46,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:46,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029286185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:46,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:46,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:46,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:46,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029286226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:46,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:46,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029286226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029286226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:46,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:46,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029286235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:46,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:46,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/88fa628b233844c385103c8ee86cfbd2 2024-11-19T15:13:46,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e1cdd24fbf874bdc95f3f0f1e475c57c is 50, key is test_row_0/C:col10/1732029225717/Put/seqid=0 2024-11-19T15:13:46,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:46,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:46,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T15:13:46,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741842_1018 (size=12001) 2024-11-19T15:13:46,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e1cdd24fbf874bdc95f3f0f1e475c57c 2024-11-19T15:13:46,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/c33944b29bdc40b49b4c01fe9681f9ad as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/c33944b29bdc40b49b4c01fe9681f9ad 2024-11-19T15:13:46,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/c33944b29bdc40b49b4c01fe9681f9ad, entries=200, sequenceid=50, filesize=14.0 K 2024-11-19T15:13:46,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/88fa628b233844c385103c8ee86cfbd2 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/88fa628b233844c385103c8ee86cfbd2 2024-11-19T15:13:46,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/88fa628b233844c385103c8ee86cfbd2, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T15:13:46,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e1cdd24fbf874bdc95f3f0f1e475c57c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e1cdd24fbf874bdc95f3f0f1e475c57c 2024-11-19T15:13:46,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e1cdd24fbf874bdc95f3f0f1e475c57c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T15:13:46,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 9a10ee694987b867f252a8f623c69480 in 770ms, sequenceid=50, compaction requested=false 2024-11-19T15:13:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:46,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:13:46,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:46,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:46,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:46,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:46,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:46,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:46,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:46,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:46,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/0ee9d2d9b4f649bfa728b8c49c5aa6cd is 50, key is test_row_0/A:col10/1732029225802/Put/seqid=0 2024-11-19T15:13:46,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741843_1019 (size=14341) 2024-11-19T15:13:46,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/0ee9d2d9b4f649bfa728b8c49c5aa6cd 2024-11-19T15:13:46,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/b7abc79f4f6444cc88887430949dc331 is 50, key is test_row_0/B:col10/1732029225802/Put/seqid=0 2024-11-19T15:13:46,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741844_1020 (size=12001) 2024-11-19T15:13:46,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/b7abc79f4f6444cc88887430949dc331 2024-11-19T15:13:46,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029226575, max timestamp=1732029226603Num uniques 859; 2024-11-19T15:13:46,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:46,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732029226575, max timestamp=1732029226605Num uniques 859; 2024-11-19T15:13:46,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T15:13:46,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732029226575, max timestamp=1732029226603Num uniques 859; 2024-11-19T15:13:46,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T15:13:46,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/5f3078325fe84054a9b0de86ae535331 is 50, key is test_row_0/C:col10/1732029225802/Put/seqid=0 2024-11-19T15:13:46,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:46,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:46,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:46,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741845_1021 (size=12001) 2024-11-19T15:13:46,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/5f3078325fe84054a9b0de86ae535331 2024-11-19T15:13:46,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/0ee9d2d9b4f649bfa728b8c49c5aa6cd as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/0ee9d2d9b4f649bfa728b8c49c5aa6cd 2024-11-19T15:13:46,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/0ee9d2d9b4f649bfa728b8c49c5aa6cd, entries=200, sequenceid=60, filesize=14.0 K 2024-11-19T15:13:46,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/b7abc79f4f6444cc88887430949dc331 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/b7abc79f4f6444cc88887430949dc331 2024-11-19T15:13:46,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/b7abc79f4f6444cc88887430949dc331, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T15:13:46,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/5f3078325fe84054a9b0de86ae535331 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5f3078325fe84054a9b0de86ae535331 2024-11-19T15:13:46,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029286681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:46,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732029226667, max timestamp=1732029226705Num uniques 1009; 2024-11-19T15:13:46,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-19T15:13:46,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732029226667, max timestamp=1732029226705Num uniques 1018; 2024-11-19T15:13:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:46,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T15:13:46,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732029226667, max timestamp=1732029226705Num uniques 1009; 2024-11-19T15:13:46,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-19T15:13:46,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5f3078325fe84054a9b0de86ae535331, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T15:13:46,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=37.22 KB/38112 for 9a10ee694987b867f252a8f623c69480 in 266ms, sequenceid=60, compaction requested=true 2024-11-19T15:13:46,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:46,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:13:46,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:46,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:13:46,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:46,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:13:46,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:46,788 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:46,788 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:46,792 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:46,794 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/B is initiating minor compaction (all files) 2024-11-19T15:13:46,794 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/B in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,795 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/78a112a56064478286aa1952674b4e35, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/88fa628b233844c385103c8ee86cfbd2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/b7abc79f4f6444cc88887430949dc331] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=35.2 K 2024-11-19T15:13:46,797 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 78a112a56064478286aa1952674b4e35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732029224216 2024-11-19T15:13:46,798 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:46,798 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 88fa628b233844c385103c8ee86cfbd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732029224583 2024-11-19T15:13:46,799 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/A is initiating minor compaction (all files) 2024-11-19T15:13:46,799 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/A in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,799 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/2b534ab50dba4556987f571fe0d28a45, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/c33944b29bdc40b49b4c01fe9681f9ad, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/0ee9d2d9b4f649bfa728b8c49c5aa6cd] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=39.7 K 2024-11-19T15:13:46,802 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting b7abc79f4f6444cc88887430949dc331, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029225802 2024-11-19T15:13:46,817 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b534ab50dba4556987f571fe0d28a45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732029224216 2024-11-19T15:13:46,818 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting c33944b29bdc40b49b4c01fe9681f9ad, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732029224583 2024-11-19T15:13:46,819 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ee9d2d9b4f649bfa728b8c49c5aa6cd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029225802 2024-11-19T15:13:46,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T15:13:46,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:46,834 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=37.22 KB heapSize=93.64 KB 2024-11-19T15:13:46,847 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#B#compaction#9 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:46,850 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/352a932d9c8641ada102ac4ce95b1eec is 50, key is test_row_0/B:col10/1732029225802/Put/seqid=0 2024-11-19T15:13:46,853 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#A#compaction#10 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:46,854 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/1553e0ce38d14fa681b2191bf8d90545 is 50, key is test_row_0/A:col10/1732029225802/Put/seqid=0 2024-11-19T15:13:46,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:46,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:46,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/0a508510351c4aa6a735244820bb7696 is 50, key is test_row_0/A:col10/1732029226760/Put/seqid=0 2024-11-19T15:13:46,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741846_1022 (size=12104) 2024-11-19T15:13:46,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741847_1023 (size=12104) 2024-11-19T15:13:46,975 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/352a932d9c8641ada102ac4ce95b1eec as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/352a932d9c8641ada102ac4ce95b1eec 2024-11-19T15:13:46,980 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/1553e0ce38d14fa681b2191bf8d90545 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1553e0ce38d14fa681b2191bf8d90545 2024-11-19T15:13:46,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741848_1024 (size=12001) 2024-11-19T15:13:47,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029226926, max timestamp=1732029226962Num uniques 859; 2024-11-19T15:13:47,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029226926, max timestamp=1732029226962Num uniques 859; 2024-11-19T15:13:47,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:47,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:47,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029226926, max timestamp=1732029226962Num uniques 859; 2024-11-19T15:13:47,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:47,028 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/B of 9a10ee694987b867f252a8f623c69480 into 352a932d9c8641ada102ac4ce95b1eec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:47,028 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:47,031 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/B, priority=13, startTime=1732029226787; duration=0sec 2024-11-19T15:13:47,032 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:47,032 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:B 2024-11-19T15:13:47,033 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:47,033 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/A of 9a10ee694987b867f252a8f623c69480 into 1553e0ce38d14fa681b2191bf8d90545(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:47,033 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:47,033 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/A, priority=13, startTime=1732029226776; duration=0sec 2024-11-19T15:13:47,033 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:47,033 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:A 2024-11-19T15:13:47,040 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:47,040 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/C is initiating minor compaction (all files) 2024-11-19T15:13:47,040 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/C in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:47,041 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/722ee79694394407b104c3be58b398fd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e1cdd24fbf874bdc95f3f0f1e475c57c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5f3078325fe84054a9b0de86ae535331] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=35.2 K 2024-11-19T15:13:47,042 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 722ee79694394407b104c3be58b398fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732029224216 2024-11-19T15:13:47,043 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting e1cdd24fbf874bdc95f3f0f1e475c57c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732029224583 2024-11-19T15:13:47,043 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f3078325fe84054a9b0de86ae535331, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029225802 2024-11-19T15:13:47,070 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#C#compaction#12 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:47,071 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/c9e4924715f84e9f830a080ccfffd2c4 is 50, key is test_row_0/C:col10/1732029225802/Put/seqid=0 2024-11-19T15:13:47,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029287075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029287084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029287089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741849_1025 (size=12104) 2024-11-19T15:13:47,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029287094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029287096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,112 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/c9e4924715f84e9f830a080ccfffd2c4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/c9e4924715f84e9f830a080ccfffd2c4 2024-11-19T15:13:47,127 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/C of 9a10ee694987b867f252a8f623c69480 into c9e4924715f84e9f830a080ccfffd2c4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:47,127 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:47,127 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/C, priority=13, startTime=1732029226787; duration=0sec 2024-11-19T15:13:47,128 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:47,128 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:C 2024-11-19T15:13:47,201 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T15:13:47,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029287204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029287205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029287205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029287206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029287215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,395 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/0a508510351c4aa6a735244820bb7696 2024-11-19T15:13:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/837720a0f21342bbabf42a0b6d34b31d is 50, key is test_row_0/B:col10/1732029226760/Put/seqid=0 2024-11-19T15:13:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029287415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029287415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029287415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029287425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029287435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741850_1026 (size=12001) 2024-11-19T15:13:47,448 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/837720a0f21342bbabf42a0b6d34b31d 2024-11-19T15:13:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/2e09255af2af4c89b37329e3d762eda7 is 50, key is test_row_0/C:col10/1732029226760/Put/seqid=0 2024-11-19T15:13:47,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741851_1027 (size=12001) 2024-11-19T15:13:47,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029287735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029287735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029287736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029287746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029287746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:47,898 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.09 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/2e09255af2af4c89b37329e3d762eda7 2024-11-19T15:13:47,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/0a508510351c4aa6a735244820bb7696 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/0a508510351c4aa6a735244820bb7696 2024-11-19T15:13:47,930 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/0a508510351c4aa6a735244820bb7696, entries=150, sequenceid=102, filesize=11.7 K 2024-11-19T15:13:47,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/837720a0f21342bbabf42a0b6d34b31d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/837720a0f21342bbabf42a0b6d34b31d 2024-11-19T15:13:47,944 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/837720a0f21342bbabf42a0b6d34b31d, entries=150, sequenceid=102, filesize=11.7 K 2024-11-19T15:13:47,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/2e09255af2af4c89b37329e3d762eda7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2e09255af2af4c89b37329e3d762eda7 2024-11-19T15:13:47,962 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2e09255af2af4c89b37329e3d762eda7, entries=150, sequenceid=102, filesize=11.7 K 2024-11-19T15:13:47,965 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~57.35 KB/58722, heapSize ~153.36 KB/157040, currentSize=134.18 KB/137400 for 9a10ee694987b867f252a8f623c69480 in 1130ms, sequenceid=102, compaction requested=false 2024-11-19T15:13:47,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:47,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:47,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-19T15:13:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-19T15:13:47,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T15:13:47,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7440 sec 2024-11-19T15:13:47,978 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T15:13:47,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.7800 sec 2024-11-19T15:13:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:48,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=154.31 KB heapSize=445.45 KB 2024-11-19T15:13:48,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:48,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T15:13:48,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:48,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T15:13:48,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:48,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T15:13:48,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/cb898b3c091c4706969eb1e4dc5fbcde is 50, key is test_row_0/A:col10/1732029228258/Put/seqid=0 2024-11-19T15:13:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029288275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029288287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029288287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029288292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029288292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741852_1028 (size=12151) 2024-11-19T15:13:48,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/cb898b3c091c4706969eb1e4dc5fbcde 2024-11-19T15:13:48,329 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T15:13:48,330 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T15:13:48,333 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T15:13:48,333 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T15:13:48,334 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T15:13:48,334 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T15:13:48,335 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T15:13:48,335 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T15:13:48,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/6fced96c4dbb4b0b8d3ab92e719c134b is 50, key is test_row_0/B:col10/1732029228258/Put/seqid=0 2024-11-19T15:13:48,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741853_1029 (size=12151) 2024-11-19T15:13:48,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/6fced96c4dbb4b0b8d3ab92e719c134b 2024-11-19T15:13:48,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/613a1dc10133471aaa95f660395b369a is 50, key is test_row_0/C:col10/1732029228258/Put/seqid=0 2024-11-19T15:13:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T15:13:48,385 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:13:48,386 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:13:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-19T15:13:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T15:13:48,390 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:13:48,391 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:13:48,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:13:48,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741854_1030 (size=12151) 2024-11-19T15:13:48,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/613a1dc10133471aaa95f660395b369a 2024-11-19T15:13:48,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029288404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029288405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029288405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029288405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029288405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/cb898b3c091c4706969eb1e4dc5fbcde as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/cb898b3c091c4706969eb1e4dc5fbcde 2024-11-19T15:13:48,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/cb898b3c091c4706969eb1e4dc5fbcde, entries=150, sequenceid=145, filesize=11.9 K 2024-11-19T15:13:48,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/6fced96c4dbb4b0b8d3ab92e719c134b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/6fced96c4dbb4b0b8d3ab92e719c134b 2024-11-19T15:13:48,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/6fced96c4dbb4b0b8d3ab92e719c134b, entries=150, sequenceid=145, filesize=11.9 K 2024-11-19T15:13:48,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/613a1dc10133471aaa95f660395b369a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/613a1dc10133471aaa95f660395b369a 2024-11-19T15:13:48,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/613a1dc10133471aaa95f660395b369a, entries=150, sequenceid=145, filesize=11.9 K 2024-11-19T15:13:48,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~445.41 KB/456096, currentSize=26.84 KB/27480 for 9a10ee694987b867f252a8f623c69480 in 200ms, sequenceid=145, compaction requested=true 2024-11-19T15:13:48,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:48,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:13:48,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:48,473 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:48,473 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:48,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:13:48,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:48,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:13:48,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:48,482 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:48,482 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/B is initiating minor compaction (all files) 2024-11-19T15:13:48,482 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/B in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:48,482 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/352a932d9c8641ada102ac4ce95b1eec, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/837720a0f21342bbabf42a0b6d34b31d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/6fced96c4dbb4b0b8d3ab92e719c134b] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=35.4 K 2024-11-19T15:13:48,483 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:48,483 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/A is initiating minor compaction (all files) 2024-11-19T15:13:48,483 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/A in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:48,483 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1553e0ce38d14fa681b2191bf8d90545, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/0a508510351c4aa6a735244820bb7696, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/cb898b3c091c4706969eb1e4dc5fbcde] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=35.4 K 2024-11-19T15:13:48,485 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 352a932d9c8641ada102ac4ce95b1eec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029225802 2024-11-19T15:13:48,486 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 837720a0f21342bbabf42a0b6d34b31d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732029226694 2024-11-19T15:13:48,486 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1553e0ce38d14fa681b2191bf8d90545, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029225802 2024-11-19T15:13:48,487 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a508510351c4aa6a735244820bb7696, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732029226694 2024-11-19T15:13:48,487 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fced96c4dbb4b0b8d3ab92e719c134b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732029227081 2024-11-19T15:13:48,488 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb898b3c091c4706969eb1e4dc5fbcde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732029227081 2024-11-19T15:13:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T15:13:48,515 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#B#compaction#18 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:48,516 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/78ab13b66687429294c6886b1f461300 is 50, key is test_row_0/B:col10/1732029228258/Put/seqid=0 2024-11-19T15:13:48,517 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#A#compaction#19 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:48,518 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/b71df9daf2754ad68695416ed52532f2 is 50, key is test_row_0/A:col10/1732029228258/Put/seqid=0 2024-11-19T15:13:48,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741855_1031 (size=12357) 2024-11-19T15:13:48,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741856_1032 (size=12357) 2024-11-19T15:13:48,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T15:13:48,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:48,548 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T15:13:48,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:48,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:48,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:48,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:48,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:48,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:48,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/5d4c7ab4d08d455fa9c5b14bf03a207e is 50, key is test_row_0/A:col10/1732029228277/Put/seqid=0 2024-11-19T15:13:48,558 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/78ab13b66687429294c6886b1f461300 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/78ab13b66687429294c6886b1f461300 2024-11-19T15:13:48,577 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/B of 9a10ee694987b867f252a8f623c69480 into 78ab13b66687429294c6886b1f461300(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:48,577 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:48,577 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/B, priority=13, startTime=1732029228473; duration=0sec 2024-11-19T15:13:48,578 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:48,578 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:B 2024-11-19T15:13:48,578 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:48,582 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:48,582 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/C is initiating minor compaction (all files) 2024-11-19T15:13:48,583 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/C in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:48,583 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/c9e4924715f84e9f830a080ccfffd2c4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2e09255af2af4c89b37329e3d762eda7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/613a1dc10133471aaa95f660395b369a] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=35.4 K 2024-11-19T15:13:48,584 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting c9e4924715f84e9f830a080ccfffd2c4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029225802 2024-11-19T15:13:48,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741857_1033 (size=9757) 2024-11-19T15:13:48,586 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e09255af2af4c89b37329e3d762eda7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732029226694 2024-11-19T15:13:48,587 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 613a1dc10133471aaa95f660395b369a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732029227081 2024-11-19T15:13:48,595 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/5d4c7ab4d08d455fa9c5b14bf03a207e 2024-11-19T15:13:48,611 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#C#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:48,614 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/dcb4426516504e94a7155b253a254574 is 50, key is test_row_0/C:col10/1732029228258/Put/seqid=0 2024-11-19T15:13:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:48,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:48,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/d1b0d86c0b654392a06e3892c71ff7b9 is 50, key is test_row_0/B:col10/1732029228277/Put/seqid=0 2024-11-19T15:13:48,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741859_1035 (size=9757) 2024-11-19T15:13:48,667 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/d1b0d86c0b654392a06e3892c71ff7b9 2024-11-19T15:13:48,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741858_1034 (size=12357) 2024-11-19T15:13:48,690 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/dcb4426516504e94a7155b253a254574 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/dcb4426516504e94a7155b253a254574 2024-11-19T15:13:48,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/2adda2c1a4c64535a5c566d5da390a64 is 50, key is test_row_0/C:col10/1732029228277/Put/seqid=0 2024-11-19T15:13:48,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732029228667, max timestamp=1732029228691Num uniques 859; 2024-11-19T15:13:48,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T15:13:48,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732029228667, max timestamp=1732029228691Num uniques 859; 2024-11-19T15:13:48,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732029228667, max timestamp=1732029228691Num uniques 859; 2024-11-19T15:13:48,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T15:13:48,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T15:13:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T15:13:48,712 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/C of 9a10ee694987b867f252a8f623c69480 into dcb4426516504e94a7155b253a254574(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:48,712 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:48,712 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/C, priority=13, startTime=1732029228474; duration=0sec 2024-11-19T15:13:48,712 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:48,712 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:C 2024-11-19T15:13:48,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741860_1036 (size=9757) 2024-11-19T15:13:48,725 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/2adda2c1a4c64535a5c566d5da390a64 2024-11-19T15:13:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/5d4c7ab4d08d455fa9c5b14bf03a207e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/5d4c7ab4d08d455fa9c5b14bf03a207e 2024-11-19T15:13:48,757 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/5d4c7ab4d08d455fa9c5b14bf03a207e, entries=100, sequenceid=152, filesize=9.5 K 2024-11-19T15:13:48,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/d1b0d86c0b654392a06e3892c71ff7b9 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/d1b0d86c0b654392a06e3892c71ff7b9 2024-11-19T15:13:48,768 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/d1b0d86c0b654392a06e3892c71ff7b9, entries=100, sequenceid=152, filesize=9.5 K 2024-11-19T15:13:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/2adda2c1a4c64535a5c566d5da390a64 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2adda2c1a4c64535a5c566d5da390a64 2024-11-19T15:13:48,778 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2adda2c1a4c64535a5c566d5da390a64, entries=100, sequenceid=152, filesize=9.5 K 2024-11-19T15:13:48,787 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=120.76 KB/123660 for 9a10ee694987b867f252a8f623c69480 in 239ms, sequenceid=152, compaction requested=false 2024-11-19T15:13:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:48,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-19T15:13:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-19T15:13:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:48,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-19T15:13:48,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:48,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:48,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:48,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:48,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:48,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:48,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T15:13:48,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 398 msec 2024-11-19T15:13:48,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 413 msec 2024-11-19T15:13:48,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/e66ab7a7b75c473498dfc6d9522137b6 is 50, key is test_row_0/A:col10/1732029228740/Put/seqid=0 2024-11-19T15:13:48,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029288816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029288817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029288821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029288826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029288826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741861_1037 (size=14541) 2024-11-19T15:13:48,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/e66ab7a7b75c473498dfc6d9522137b6 2024-11-19T15:13:48,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/4714a78bb4a549d8b6729995450fb1c5 is 50, key is test_row_0/B:col10/1732029228740/Put/seqid=0 2024-11-19T15:13:48,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741862_1038 (size=12151) 2024-11-19T15:13:48,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/4714a78bb4a549d8b6729995450fb1c5 2024-11-19T15:13:48,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/4145b77b37a24be6b6553931365b966f is 50, key is test_row_0/C:col10/1732029228740/Put/seqid=0 2024-11-19T15:13:48,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029288934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029288935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029288935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029288945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029288945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:48,965 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/b71df9daf2754ad68695416ed52532f2 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/b71df9daf2754ad68695416ed52532f2 2024-11-19T15:13:48,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741863_1039 (size=12151) 2024-11-19T15:13:48,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/4145b77b37a24be6b6553931365b966f 2024-11-19T15:13:48,982 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/A of 9a10ee694987b867f252a8f623c69480 into b71df9daf2754ad68695416ed52532f2(size=12.1 K), total size for store is 21.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:48,982 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:48,982 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/A, priority=13, startTime=1732029228473; duration=0sec 2024-11-19T15:13:48,983 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:48,983 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:A 2024-11-19T15:13:48,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/e66ab7a7b75c473498dfc6d9522137b6 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/e66ab7a7b75c473498dfc6d9522137b6 2024-11-19T15:13:48,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/e66ab7a7b75c473498dfc6d9522137b6, entries=200, sequenceid=189, filesize=14.2 K 2024-11-19T15:13:49,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/4714a78bb4a549d8b6729995450fb1c5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/4714a78bb4a549d8b6729995450fb1c5 2024-11-19T15:13:49,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/4714a78bb4a549d8b6729995450fb1c5, entries=150, sequenceid=189, filesize=11.9 K 2024-11-19T15:13:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T15:13:49,014 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:13:49,015 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:13:49,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/4145b77b37a24be6b6553931365b966f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/4145b77b37a24be6b6553931365b966f 2024-11-19T15:13:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-19T15:13:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T15:13:49,018 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:13:49,020 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:13:49,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:13:49,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/4145b77b37a24be6b6553931365b966f, entries=150, sequenceid=189, filesize=11.9 K 2024-11-19T15:13:49,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 9a10ee694987b867f252a8f623c69480 in 246ms, sequenceid=189, compaction requested=true 2024-11-19T15:13:49,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:49,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:13:49,037 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:49,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:49,037 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:49,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:13:49,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:49,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:13:49,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:49,044 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:49,045 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/A is initiating minor compaction (all files) 2024-11-19T15:13:49,045 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/A in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:49,045 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/b71df9daf2754ad68695416ed52532f2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/5d4c7ab4d08d455fa9c5b14bf03a207e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/e66ab7a7b75c473498dfc6d9522137b6] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=35.8 K 2024-11-19T15:13:49,045 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:49,045 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/B is initiating minor compaction (all files) 2024-11-19T15:13:49,046 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/B in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:49,046 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/78ab13b66687429294c6886b1f461300, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/d1b0d86c0b654392a06e3892c71ff7b9, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/4714a78bb4a549d8b6729995450fb1c5] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=33.5 K 2024-11-19T15:13:49,046 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting b71df9daf2754ad68695416ed52532f2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732029227081 2024-11-19T15:13:49,047 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 78ab13b66687429294c6886b1f461300, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732029227081 2024-11-19T15:13:49,048 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d4c7ab4d08d455fa9c5b14bf03a207e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732029228277 2024-11-19T15:13:49,048 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d1b0d86c0b654392a06e3892c71ff7b9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732029228277 2024-11-19T15:13:49,048 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e66ab7a7b75c473498dfc6d9522137b6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732029228723 2024-11-19T15:13:49,050 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 4714a78bb4a549d8b6729995450fb1c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732029228723 2024-11-19T15:13:49,071 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#B#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:49,072 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/ad650fea4c6c442aa2fe71fffc4912de is 50, key is test_row_0/B:col10/1732029228740/Put/seqid=0 2024-11-19T15:13:49,079 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#A#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:49,080 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/6a5b02a22a4e49f7b4ae7fdfbca4f249 is 50, key is test_row_0/A:col10/1732029228740/Put/seqid=0 2024-11-19T15:13:49,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741864_1040 (size=12459) 2024-11-19T15:13:49,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T15:13:49,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741865_1041 (size=12459) 2024-11-19T15:13:49,151 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/6a5b02a22a4e49f7b4ae7fdfbca4f249 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/6a5b02a22a4e49f7b4ae7fdfbca4f249 2024-11-19T15:13:49,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T15:13:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:49,179 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:13:49,202 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/A of 9a10ee694987b867f252a8f623c69480 into 6a5b02a22a4e49f7b4ae7fdfbca4f249(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:49,202 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:49,202 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/A, priority=13, startTime=1732029229037; duration=0sec 2024-11-19T15:13:49,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:49,202 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:49,202 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:A 2024-11-19T15:13:49,202 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:49,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:49,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:49,205 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:49,205 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/C is initiating minor compaction (all files) 2024-11-19T15:13:49,205 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/C in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:49,205 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/dcb4426516504e94a7155b253a254574, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2adda2c1a4c64535a5c566d5da390a64, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/4145b77b37a24be6b6553931365b966f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=33.5 K 2024-11-19T15:13:49,206 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcb4426516504e94a7155b253a254574, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732029227081 2024-11-19T15:13:49,212 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2adda2c1a4c64535a5c566d5da390a64, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732029228277 2024-11-19T15:13:49,213 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4145b77b37a24be6b6553931365b966f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732029228723 2024-11-19T15:13:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/60798c3f1ddc40d3bfec465e1334ba6b is 50, key is test_row_0/A:col10/1732029229162/Put/seqid=0 2024-11-19T15:13:49,241 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:49,242 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/1744cf6d1ecd4f1abb44ec45f64822da is 50, key is test_row_0/C:col10/1732029228740/Put/seqid=0 2024-11-19T15:13:49,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741866_1042 (size=12151) 2024-11-19T15:13:49,247 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/60798c3f1ddc40d3bfec465e1334ba6b 2024-11-19T15:13:49,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741867_1043 (size=12459) 2024-11-19T15:13:49,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/8e95f6148e5f4b00ae4517b1cd0552ca is 50, key is test_row_0/B:col10/1732029229162/Put/seqid=0 2024-11-19T15:13:49,271 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/1744cf6d1ecd4f1abb44ec45f64822da as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/1744cf6d1ecd4f1abb44ec45f64822da 2024-11-19T15:13:49,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029289246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029289252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029289252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029289252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029289256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,284 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/C of 9a10ee694987b867f252a8f623c69480 into 1744cf6d1ecd4f1abb44ec45f64822da(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:49,285 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:49,285 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/C, priority=13, startTime=1732029229038; duration=0sec 2024-11-19T15:13:49,285 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:49,285 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:C 2024-11-19T15:13:49,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741868_1044 (size=12151) 2024-11-19T15:13:49,304 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/8e95f6148e5f4b00ae4517b1cd0552ca 2024-11-19T15:13:49,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/fc4e78e551ee4996a765f66de9263723 is 50, key is test_row_0/C:col10/1732029229162/Put/seqid=0 2024-11-19T15:13:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T15:13:49,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741869_1045 (size=12151) 2024-11-19T15:13:49,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029289385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029289385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029289385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029289386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029289386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,535 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/ad650fea4c6c442aa2fe71fffc4912de as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/ad650fea4c6c442aa2fe71fffc4912de 2024-11-19T15:13:49,549 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/B of 9a10ee694987b867f252a8f623c69480 into ad650fea4c6c442aa2fe71fffc4912de(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:49,549 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:49,549 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/B, priority=13, startTime=1732029229037; duration=0sec 2024-11-19T15:13:49,550 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:49,550 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:B 2024-11-19T15:13:49,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029289595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029289595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029289595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029289595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029289596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T15:13:49,753 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/fc4e78e551ee4996a765f66de9263723 2024-11-19T15:13:49,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/60798c3f1ddc40d3bfec465e1334ba6b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/60798c3f1ddc40d3bfec465e1334ba6b 2024-11-19T15:13:49,775 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/60798c3f1ddc40d3bfec465e1334ba6b, entries=150, sequenceid=204, filesize=11.9 K 2024-11-19T15:13:49,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/8e95f6148e5f4b00ae4517b1cd0552ca as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/8e95f6148e5f4b00ae4517b1cd0552ca 2024-11-19T15:13:49,783 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/8e95f6148e5f4b00ae4517b1cd0552ca, entries=150, sequenceid=204, filesize=11.9 K 2024-11-19T15:13:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/fc4e78e551ee4996a765f66de9263723 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/fc4e78e551ee4996a765f66de9263723 2024-11-19T15:13:49,793 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/fc4e78e551ee4996a765f66de9263723, entries=150, sequenceid=204, filesize=11.9 K 2024-11-19T15:13:49,794 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9a10ee694987b867f252a8f623c69480 in 615ms, sequenceid=204, compaction requested=false 2024-11-19T15:13:49,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:49,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:49,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-19T15:13:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-19T15:13:49,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T15:13:49,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 777 msec 2024-11-19T15:13:49,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 788 msec 2024-11-19T15:13:49,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T15:13:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:49,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732029229239, max timestamp=1732029229908Num uniques 859; 2024-11-19T15:13:49,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732029229239, max timestamp=1732029229908Num uniques 859; 2024-11-19T15:13:49,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732029229239, max timestamp=1732029229908Num uniques 859; 2024-11-19T15:13:49,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T15:13:49,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T15:13:49,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T15:13:49,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:49,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:49,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:49,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:49,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:49,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:49,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/30db31b174cc4f199ac4d1181241274a is 50, key is test_row_0/A:col10/1732029229910/Put/seqid=0 2024-11-19T15:13:49,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741870_1046 (size=16931) 2024-11-19T15:13:49,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/30db31b174cc4f199ac4d1181241274a 2024-11-19T15:13:49,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732029229939, max timestamp=1732029229957Num uniques 859; 2024-11-19T15:13:49,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T15:13:49,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732029229939, max timestamp=1732029229956Num uniques 859; 2024-11-19T15:13:49,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T15:13:49,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732029229939, max timestamp=1732029229957Num uniques 859; 2024-11-19T15:13:49,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T15:13:49,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/57324c53aee549b4aa3be88536849757 is 50, key is test_row_0/B:col10/1732029229910/Put/seqid=0 2024-11-19T15:13:49,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741871_1047 (size=12151) 2024-11-19T15:13:50,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029289994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029289995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029290002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029290003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029290005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029290115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029290115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029290125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029290125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029290125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T15:13:50,154 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:13:50,155 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:13:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-19T15:13:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T15:13:50,158 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:13:50,159 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:13:50,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:13:50,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T15:13:50,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T15:13:50,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:50,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:50,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:50,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:50,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:50,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:50,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029290334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029290334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029290344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029290345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029290345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/57324c53aee549b4aa3be88536849757 2024-11-19T15:13:50,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/de15a198e73941e9a1d1eaf370a34e54 is 50, key is test_row_0/C:col10/1732029229910/Put/seqid=0 2024-11-19T15:13:50,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741872_1048 (size=12151) 2024-11-19T15:13:50,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T15:13:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:50,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:50,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:50,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T15:13:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:50,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T15:13:50,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:50,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:50,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029290645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029290645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029290655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029290655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029290664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T15:13:50,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T15:13:50,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:50,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:50,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:50,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:50,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:50,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/de15a198e73941e9a1d1eaf370a34e54 2024-11-19T15:13:50,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/30db31b174cc4f199ac4d1181241274a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/30db31b174cc4f199ac4d1181241274a 2024-11-19T15:13:50,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/30db31b174cc4f199ac4d1181241274a, entries=250, sequenceid=229, filesize=16.5 K 2024-11-19T15:13:50,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/57324c53aee549b4aa3be88536849757 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/57324c53aee549b4aa3be88536849757 2024-11-19T15:13:50,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/57324c53aee549b4aa3be88536849757, entries=150, sequenceid=229, filesize=11.9 K 2024-11-19T15:13:50,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/de15a198e73941e9a1d1eaf370a34e54 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/de15a198e73941e9a1d1eaf370a34e54 2024-11-19T15:13:50,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/de15a198e73941e9a1d1eaf370a34e54, entries=150, sequenceid=229, filesize=11.9 K 2024-11-19T15:13:50,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.07 KB/40008, heapSize ~103.29 KB/105768, currentSize=161.02 KB/164880 for 9a10ee694987b867f252a8f623c69480 in 972ms, sequenceid=229, compaction requested=true 2024-11-19T15:13:50,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:50,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:13:50,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:50,882 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:50,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:13:50,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:50,882 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:50,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:13:50,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:50,884 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41541 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:50,885 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/A is initiating minor compaction (all files) 2024-11-19T15:13:50,885 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/A in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:50,885 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/6a5b02a22a4e49f7b4ae7fdfbca4f249, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/60798c3f1ddc40d3bfec465e1334ba6b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/30db31b174cc4f199ac4d1181241274a] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=40.6 K 2024-11-19T15:13:50,885 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:50,885 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/B is initiating minor compaction (all files) 2024-11-19T15:13:50,885 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/B in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:50,885 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a5b02a22a4e49f7b4ae7fdfbca4f249, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732029228723 2024-11-19T15:13:50,886 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/ad650fea4c6c442aa2fe71fffc4912de, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/8e95f6148e5f4b00ae4517b1cd0552ca, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/57324c53aee549b4aa3be88536849757] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=35.9 K 2024-11-19T15:13:50,886 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting ad650fea4c6c442aa2fe71fffc4912de, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732029228723 2024-11-19T15:13:50,886 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60798c3f1ddc40d3bfec465e1334ba6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732029228813 2024-11-19T15:13:50,887 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30db31b174cc4f199ac4d1181241274a, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732029229239 2024-11-19T15:13:50,887 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e95f6148e5f4b00ae4517b1cd0552ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732029228813 2024-11-19T15:13:50,887 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 57324c53aee549b4aa3be88536849757, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732029229908 2024-11-19T15:13:50,903 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:50,904 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/08db943566d340b9b97fb14f98c55473 is 50, key is test_row_0/B:col10/1732029229910/Put/seqid=0 2024-11-19T15:13:50,907 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:50,908 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/eb628ca0f5e744e9abe685be08d214e1 is 50, key is test_row_0/A:col10/1732029229910/Put/seqid=0 2024-11-19T15:13:50,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741874_1050 (size=12561) 2024-11-19T15:13:50,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T15:13:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:50,948 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=161.02 KB heapSize=445.92 KB 2024-11-19T15:13:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:50,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:50,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:50,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:50,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:50,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741873_1049 (size=12561) 2024-11-19T15:13:50,954 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/08db943566d340b9b97fb14f98c55473 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/08db943566d340b9b97fb14f98c55473 2024-11-19T15:13:50,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/935d72b396ef4e64a3770b31e4d22d8f is 50, key is test_row_0/A:col10/1732029229985/Put/seqid=0 2024-11-19T15:13:50,967 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/eb628ca0f5e744e9abe685be08d214e1 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/eb628ca0f5e744e9abe685be08d214e1 2024-11-19T15:13:50,972 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/B of 9a10ee694987b867f252a8f623c69480 into 08db943566d340b9b97fb14f98c55473(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:50,972 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:50,972 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/B, priority=13, startTime=1732029230882; duration=0sec 2024-11-19T15:13:50,972 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:50,972 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:B 2024-11-19T15:13:50,972 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:50,974 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:50,974 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/C is initiating minor compaction (all files) 2024-11-19T15:13:50,974 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/C in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:50,974 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/1744cf6d1ecd4f1abb44ec45f64822da, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/fc4e78e551ee4996a765f66de9263723, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/de15a198e73941e9a1d1eaf370a34e54] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=35.9 K 2024-11-19T15:13:50,975 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 1744cf6d1ecd4f1abb44ec45f64822da, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732029228723 2024-11-19T15:13:50,976 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/A of 9a10ee694987b867f252a8f623c69480 into eb628ca0f5e744e9abe685be08d214e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:50,976 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:50,976 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/A, priority=13, startTime=1732029230882; duration=0sec 2024-11-19T15:13:50,977 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:50,977 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:A 2024-11-19T15:13:50,975 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting fc4e78e551ee4996a765f66de9263723, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732029228813 2024-11-19T15:13:50,977 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting de15a198e73941e9a1d1eaf370a34e54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732029229908 2024-11-19T15:13:50,992 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:50,993 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/93c60c3182c8451686c4cee891cfebf6 is 50, key is test_row_0/C:col10/1732029229910/Put/seqid=0 2024-11-19T15:13:50,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741875_1051 (size=12301) 2024-11-19T15:13:50,996 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/935d72b396ef4e64a3770b31e4d22d8f 2024-11-19T15:13:51,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741876_1052 (size=12561) 2024-11-19T15:13:51,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/65d6af4ef3f14b359485ae7a489ea3eb is 50, key is test_row_0/B:col10/1732029229985/Put/seqid=0 2024-11-19T15:13:51,030 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/93c60c3182c8451686c4cee891cfebf6 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/93c60c3182c8451686c4cee891cfebf6 2024-11-19T15:13:51,040 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/C of 9a10ee694987b867f252a8f623c69480 into 93c60c3182c8451686c4cee891cfebf6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:51,040 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:51,040 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/C, priority=13, startTime=1732029230882; duration=0sec 2024-11-19T15:13:51,041 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:51,041 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:C 2024-11-19T15:13:51,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741877_1053 (size=12301) 2024-11-19T15:13:51,044 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/65d6af4ef3f14b359485ae7a489ea3eb 2024-11-19T15:13:51,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/5fc0379eba994831a6fbe808c2d62888 is 50, key is test_row_0/C:col10/1732029229985/Put/seqid=0 2024-11-19T15:13:51,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741878_1054 (size=12301) 2024-11-19T15:13:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:51,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:51,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029291165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:51,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029291166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:51,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029291175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:51,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029291175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029291175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:51,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029291274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:51,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029291275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T15:13:51,472 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/5fc0379eba994831a6fbe808c2d62888 2024-11-19T15:13:51,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/935d72b396ef4e64a3770b31e4d22d8f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/935d72b396ef4e64a3770b31e4d22d8f 2024-11-19T15:13:51,494 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/935d72b396ef4e64a3770b31e4d22d8f, entries=150, sequenceid=266, filesize=12.0 K 2024-11-19T15:13:51,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/65d6af4ef3f14b359485ae7a489ea3eb as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/65d6af4ef3f14b359485ae7a489ea3eb 2024-11-19T15:13:51,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029291494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:51,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029291494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:51,509 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/65d6af4ef3f14b359485ae7a489ea3eb, entries=150, sequenceid=266, filesize=12.0 K 2024-11-19T15:13:51,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/5fc0379eba994831a6fbe808c2d62888 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5fc0379eba994831a6fbe808c2d62888 2024-11-19T15:13:51,520 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5fc0379eba994831a6fbe808c2d62888, entries=150, sequenceid=266, filesize=12.0 K 2024-11-19T15:13:51,521 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.88 KB/456576, currentSize=26.84 KB/27480 for 9a10ee694987b867f252a8f623c69480 in 573ms, sequenceid=266, compaction requested=false 2024-11-19T15:13:51,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:51,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:51,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-19T15:13:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-19T15:13:51,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T15:13:51,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3650 sec 2024-11-19T15:13:51,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3750 sec 2024-11-19T15:13:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:51,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:13:51,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:51,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/d817fda905154e67863a8196521a13e4 is 50, key is test_row_0/A:col10/1732029231820/Put/seqid=0 2024-11-19T15:13:51,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741879_1055 (size=14737) 2024-11-19T15:13:51,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732029231885, max timestamp=1732029231907Num uniques 859; 2024-11-19T15:13:51,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732029231885, max timestamp=1732029231907Num uniques 859; 2024-11-19T15:13:51,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T15:13:51,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T15:13:51,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732029231885, max timestamp=1732029231907Num uniques 859; 2024-11-19T15:13:51,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T15:13:51,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029291966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:51,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029291966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:51,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:52,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:52,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029292084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029292084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:52,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029292184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:52,191 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in 9a10ee694987b867f252a8f623c69480 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:13:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:52,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029292195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:52,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029292195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:52,196 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 9a10ee694987b867f252a8f623c69480 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:13:52,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:52,196 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 9a10ee694987b867f252a8f623c69480 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:13:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:52,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/d817fda905154e67863a8196521a13e4 2024-11-19T15:13:52,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/8d0f20c8ce734e5d9eb7d3833ff9e02f is 50, key is test_row_0/B:col10/1732029231820/Put/seqid=0 2024-11-19T15:13:52,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741880_1056 (size=9857) 2024-11-19T15:13:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T15:13:52,311 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:13:52,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029292310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:52,312 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:13:52,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029292310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-19T15:13:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T15:13:52,314 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:13:52,315 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:13:52,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:13:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T15:13:52,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T15:13:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:52,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:52,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T15:13:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:52,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029292624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029292624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T15:13:52,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/8d0f20c8ce734e5d9eb7d3833ff9e02f 2024-11-19T15:13:52,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e8e10e97a0ba412d9cbc3a256a37a637 is 50, key is test_row_0/C:col10/1732029231820/Put/seqid=0 2024-11-19T15:13:52,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741881_1057 (size=9857) 2024-11-19T15:13:52,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e8e10e97a0ba412d9cbc3a256a37a637 2024-11-19T15:13:52,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/d817fda905154e67863a8196521a13e4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/d817fda905154e67863a8196521a13e4 2024-11-19T15:13:52,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/d817fda905154e67863a8196521a13e4, entries=200, sequenceid=280, filesize=14.4 K 2024-11-19T15:13:52,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/8d0f20c8ce734e5d9eb7d3833ff9e02f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/8d0f20c8ce734e5d9eb7d3833ff9e02f 2024-11-19T15:13:52,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/8d0f20c8ce734e5d9eb7d3833ff9e02f, entries=100, sequenceid=280, filesize=9.6 K 2024-11-19T15:13:52,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e8e10e97a0ba412d9cbc3a256a37a637 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e8e10e97a0ba412d9cbc3a256a37a637 2024-11-19T15:13:52,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e8e10e97a0ba412d9cbc3a256a37a637, entries=100, sequenceid=280, filesize=9.6 K 2024-11-19T15:13:52,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 9a10ee694987b867f252a8f623c69480 in 936ms, sequenceid=280, compaction requested=true 2024-11-19T15:13:52,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:52,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:13:52,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:52,759 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:52,759 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:52,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:13:52,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:52,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:13:52,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:52,761 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:52,761 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/A is initiating minor compaction (all files) 2024-11-19T15:13:52,761 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/A in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:52,762 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/eb628ca0f5e744e9abe685be08d214e1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/935d72b396ef4e64a3770b31e4d22d8f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/d817fda905154e67863a8196521a13e4] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=38.7 K 2024-11-19T15:13:52,762 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:52,762 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/B is initiating minor compaction (all files) 2024-11-19T15:13:52,762 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/B in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:52,762 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb628ca0f5e744e9abe685be08d214e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732029229908 2024-11-19T15:13:52,762 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/08db943566d340b9b97fb14f98c55473, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/65d6af4ef3f14b359485ae7a489ea3eb, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/8d0f20c8ce734e5d9eb7d3833ff9e02f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=33.9 K 2024-11-19T15:13:52,763 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 935d72b396ef4e64a3770b31e4d22d8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732029229985 2024-11-19T15:13:52,763 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 08db943566d340b9b97fb14f98c55473, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732029229908 2024-11-19T15:13:52,763 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d6af4ef3f14b359485ae7a489ea3eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732029229985 2024-11-19T15:13:52,764 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting d817fda905154e67863a8196521a13e4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732029231816 2024-11-19T15:13:52,764 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d0f20c8ce734e5d9eb7d3833ff9e02f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732029231819 2024-11-19T15:13:52,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T15:13:52,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:52,779 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-19T15:13:52,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:52,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:52,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:52,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:52,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:52,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:52,780 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:52,781 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/8010018295b24ac9bde6392f49bf2587 is 50, key is test_row_0/A:col10/1732029231820/Put/seqid=0 2024-11-19T15:13:52,790 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#B#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:52,791 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/770021b7a8934656b0754192c21f828b is 50, key is test_row_0/B:col10/1732029231820/Put/seqid=0 2024-11-19T15:13:52,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/4536a3e533f0437e99ff7d7999df9c37 is 50, key is test_row_0/A:col10/1732029231964/Put/seqid=0 2024-11-19T15:13:52,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741882_1058 (size=12813) 2024-11-19T15:13:52,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741884_1060 (size=12301) 2024-11-19T15:13:52,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741883_1059 (size=12813) 2024-11-19T15:13:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T15:13:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:53,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:53,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029293156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:53,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:53,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029293160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:53,215 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/8010018295b24ac9bde6392f49bf2587 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8010018295b24ac9bde6392f49bf2587 2024-11-19T15:13:53,224 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/A of 9a10ee694987b867f252a8f623c69480 into 8010018295b24ac9bde6392f49bf2587(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:53,224 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:53,224 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/A, priority=13, startTime=1732029232759; duration=0sec 2024-11-19T15:13:53,224 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:53,224 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:A 2024-11-19T15:13:53,224 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:53,225 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/4536a3e533f0437e99ff7d7999df9c37 2024-11-19T15:13:53,226 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:53,226 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/C is initiating minor compaction (all files) 2024-11-19T15:13:53,226 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/C in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:53,226 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/93c60c3182c8451686c4cee891cfebf6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5fc0379eba994831a6fbe808c2d62888, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e8e10e97a0ba412d9cbc3a256a37a637] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=33.9 K 2024-11-19T15:13:53,227 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93c60c3182c8451686c4cee891cfebf6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732029229908 2024-11-19T15:13:53,228 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fc0379eba994831a6fbe808c2d62888, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732029229985 2024-11-19T15:13:53,229 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8e10e97a0ba412d9cbc3a256a37a637, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732029231819 2024-11-19T15:13:53,238 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/770021b7a8934656b0754192c21f828b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/770021b7a8934656b0754192c21f828b 2024-11-19T15:13:53,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/67b2f56a788d45be85d95a37c2d1d294 is 50, key is test_row_0/B:col10/1732029231964/Put/seqid=0 2024-11-19T15:13:53,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741885_1061 (size=12301) 2024-11-19T15:13:53,252 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/B of 9a10ee694987b867f252a8f623c69480 into 770021b7a8934656b0754192c21f828b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:53,252 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/67b2f56a788d45be85d95a37c2d1d294 2024-11-19T15:13:53,252 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:53,252 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/B, priority=13, startTime=1732029232759; duration=0sec 2024-11-19T15:13:53,252 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:53,252 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:B 2024-11-19T15:13:53,257 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#C#compaction#49 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:53,258 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/570af6493da04e2bb9d2dc526587cbd5 is 50, key is test_row_0/C:col10/1732029231820/Put/seqid=0 2024-11-19T15:13:53,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741886_1062 (size=12813) 2024-11-19T15:13:53,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/a395c1cc3ca6421d9f58efee8f10b3e0 is 50, key is test_row_0/C:col10/1732029231964/Put/seqid=0 2024-11-19T15:13:53,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:53,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029293274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:53,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:53,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029293274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:53,283 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/570af6493da04e2bb9d2dc526587cbd5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/570af6493da04e2bb9d2dc526587cbd5 2024-11-19T15:13:53,294 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/C of 9a10ee694987b867f252a8f623c69480 into 570af6493da04e2bb9d2dc526587cbd5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:53,294 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:53,294 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/C, priority=13, startTime=1732029232760; duration=0sec 2024-11-19T15:13:53,294 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:53,294 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:C 2024-11-19T15:13:53,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741887_1063 (size=12301) 2024-11-19T15:13:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T15:13:53,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:53,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029293485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:53,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:53,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:53,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029293494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:53,706 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/a395c1cc3ca6421d9f58efee8f10b3e0 2024-11-19T15:13:53,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/4536a3e533f0437e99ff7d7999df9c37 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/4536a3e533f0437e99ff7d7999df9c37 2024-11-19T15:13:53,725 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/4536a3e533f0437e99ff7d7999df9c37, entries=150, sequenceid=314, filesize=12.0 K 2024-11-19T15:13:53,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/67b2f56a788d45be85d95a37c2d1d294 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/67b2f56a788d45be85d95a37c2d1d294 2024-11-19T15:13:53,739 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/67b2f56a788d45be85d95a37c2d1d294, entries=150, sequenceid=314, filesize=12.0 K 2024-11-19T15:13:53,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/a395c1cc3ca6421d9f58efee8f10b3e0 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/a395c1cc3ca6421d9f58efee8f10b3e0 2024-11-19T15:13:53,747 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/a395c1cc3ca6421d9f58efee8f10b3e0, entries=150, sequenceid=314, filesize=12.0 K 2024-11-19T15:13:53,749 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 9a10ee694987b867f252a8f623c69480 in 970ms, sequenceid=314, compaction requested=false 2024-11-19T15:13:53,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:53,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:53,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-19T15:13:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-19T15:13:53,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T15:13:53,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4360 sec 2024-11-19T15:13:53,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4430 sec 2024-11-19T15:13:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:53,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T15:13:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:53,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:53,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:53,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:53,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:53,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/94becc5056ee43dd84ddf486b0bc83fe is 50, key is test_row_0/A:col10/1732029233148/Put/seqid=0 2024-11-19T15:13:53,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741888_1064 (size=14741) 2024-11-19T15:13:53,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/94becc5056ee43dd84ddf486b0bc83fe 2024-11-19T15:13:53,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/ba37445629a7479f8924a5b0b72c4c25 is 50, key is test_row_0/B:col10/1732029233148/Put/seqid=0 2024-11-19T15:13:53,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741889_1065 (size=12301) 2024-11-19T15:13:53,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029293862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:53,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029293864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:53,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029293974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029293975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:54,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029294185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:54,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029294185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:54,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029294204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:54,206 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 9a10ee694987b867f252a8f623c69480 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:13:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:54,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029294215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:54,217 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in 9a10ee694987b867f252a8f623c69480 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:13:54,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:54,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:54,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:54,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029294215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:54,219 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 9a10ee694987b867f252a8f623c69480 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:13:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:54,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/ba37445629a7479f8924a5b0b72c4c25 2024-11-19T15:13:54,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/d865538949e742438fa38c8114796f6f is 50, key is test_row_0/C:col10/1732029233148/Put/seqid=0 2024-11-19T15:13:54,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741890_1066 (size=12301) 2024-11-19T15:13:54,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/d865538949e742438fa38c8114796f6f 2024-11-19T15:13:54,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/94becc5056ee43dd84ddf486b0bc83fe as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/94becc5056ee43dd84ddf486b0bc83fe 2024-11-19T15:13:54,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/94becc5056ee43dd84ddf486b0bc83fe, entries=200, sequenceid=329, filesize=14.4 K 2024-11-19T15:13:54,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/ba37445629a7479f8924a5b0b72c4c25 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/ba37445629a7479f8924a5b0b72c4c25 2024-11-19T15:13:54,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/ba37445629a7479f8924a5b0b72c4c25, entries=150, sequenceid=329, filesize=12.0 K 2024-11-19T15:13:54,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/d865538949e742438fa38c8114796f6f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/d865538949e742438fa38c8114796f6f 2024-11-19T15:13:54,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/d865538949e742438fa38c8114796f6f, entries=150, sequenceid=329, filesize=12.0 K 2024-11-19T15:13:54,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9a10ee694987b867f252a8f623c69480 in 533ms, sequenceid=329, compaction requested=true 2024-11-19T15:13:54,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:54,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:13:54,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:54,334 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:54,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:13:54,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:54,334 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:54,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:13:54,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:54,335 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:54,335 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:54,335 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/A is initiating minor compaction (all files) 2024-11-19T15:13:54,335 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/B is initiating minor compaction (all files) 2024-11-19T15:13:54,335 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/B in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:54,335 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/770021b7a8934656b0754192c21f828b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/67b2f56a788d45be85d95a37c2d1d294, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/ba37445629a7479f8924a5b0b72c4c25] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=36.5 K 2024-11-19T15:13:54,335 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/A in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:54,336 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8010018295b24ac9bde6392f49bf2587, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/4536a3e533f0437e99ff7d7999df9c37, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/94becc5056ee43dd84ddf486b0bc83fe] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=38.9 K 2024-11-19T15:13:54,336 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 770021b7a8934656b0754192c21f828b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732029230001 2024-11-19T15:13:54,336 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8010018295b24ac9bde6392f49bf2587, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732029230001 2024-11-19T15:13:54,337 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 67b2f56a788d45be85d95a37c2d1d294, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732029231954 2024-11-19T15:13:54,338 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4536a3e533f0437e99ff7d7999df9c37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732029231954 2024-11-19T15:13:54,338 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting ba37445629a7479f8924a5b0b72c4c25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732029233148 2024-11-19T15:13:54,339 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94becc5056ee43dd84ddf486b0bc83fe, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732029233148 2024-11-19T15:13:54,351 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#A#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:54,352 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/8adc0e9942294697a8a46de6f00dbe0d is 50, key is test_row_0/A:col10/1732029233148/Put/seqid=0 2024-11-19T15:13:54,354 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:54,355 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/15fa3562efcc45a5b910dbb3e8f86c2b is 50, key is test_row_0/B:col10/1732029233148/Put/seqid=0 2024-11-19T15:13:54,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741891_1067 (size=12915) 2024-11-19T15:13:54,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741892_1068 (size=12915) 2024-11-19T15:13:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T15:13:54,464 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:13:54,465 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:13:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-19T15:13:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T15:13:54,467 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:13:54,468 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:13:54,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:13:54,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029233843, max timestamp=1732029234495Num uniques 859; 2024-11-19T15:13:54,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:54,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029233843, max timestamp=1732029234495Num uniques 859; 2024-11-19T15:13:54,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:54,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029233843, max timestamp=1732029234497Num uniques 859; 2024-11-19T15:13:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:54,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=33.54 KB heapSize=86.16 KB 2024-11-19T15:13:54,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:54,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:54,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:54,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:54,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:54,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:54,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:54,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/42d3b17fe00348b69eeff51008c21614 is 50, key is test_row_0/A:col10/1732029234495/Put/seqid=0 2024-11-19T15:13:54,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741893_1069 (size=17181) 2024-11-19T15:13:54,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/42d3b17fe00348b69eeff51008c21614 2024-11-19T15:13:54,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/df547eb2507f4d2d87a398b0b3bd5026 is 50, key is test_row_0/B:col10/1732029234495/Put/seqid=0 2024-11-19T15:13:54,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741894_1070 (size=12301) 2024-11-19T15:13:54,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/df547eb2507f4d2d87a398b0b3bd5026 2024-11-19T15:13:54,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/14dc77378b424e2ebaed0533becbfba0 is 50, key is test_row_0/C:col10/1732029234495/Put/seqid=0 2024-11-19T15:13:54,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741895_1071 (size=12301) 2024-11-19T15:13:54,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029234525, max timestamp=1732029234565Num uniques 859; 2024-11-19T15:13:54,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:54,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029234525, max timestamp=1732029234565Num uniques 859; 2024-11-19T15:13:54,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:54,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029234525, max timestamp=1732029234565Num uniques 859; 2024-11-19T15:13:54,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:13:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T15:13:54,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T15:13:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:54,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:54,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732029234593, max timestamp=1732029234626Num uniques 1018; 2024-11-19T15:13:54,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732029234593, max timestamp=1732029234626Num uniques 1018; 2024-11-19T15:13:54,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732029234593, max timestamp=1732029234626Num uniques 1018; 2024-11-19T15:13:54,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T15:13:54,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T15:13:54,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T15:13:54,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732029234653, max timestamp=1732029234684Num uniques 1027; 2024-11-19T15:13:54,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T15:13:54,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732029234653, max timestamp=1732029234684Num uniques 1027; 2024-11-19T15:13:54,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T15:13:54,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732029234653, max timestamp=1732029234684Num uniques 1027; 2024-11-19T15:13:54,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T15:13:54,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732029234706, max timestamp=1732029234743Num uniques 1036; 2024-11-19T15:13:54,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-19T15:13:54,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732029234706, max timestamp=1732029234743Num uniques 1036; 2024-11-19T15:13:54,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-19T15:13:54,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732029234736, max timestamp=1732029234743Num uniques 1036; 2024-11-19T15:13:54,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-11-19T15:13:54,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T15:13:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:54,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:54,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:54,782 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/8adc0e9942294697a8a46de6f00dbe0d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8adc0e9942294697a8a46de6f00dbe0d 2024-11-19T15:13:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T15:13:54,785 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/15fa3562efcc45a5b910dbb3e8f86c2b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/15fa3562efcc45a5b910dbb3e8f86c2b 2024-11-19T15:13:54,792 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/B of 9a10ee694987b867f252a8f623c69480 into 15fa3562efcc45a5b910dbb3e8f86c2b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:54,792 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/A of 9a10ee694987b867f252a8f623c69480 into 8adc0e9942294697a8a46de6f00dbe0d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:54,793 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:54,793 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:54,793 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/B, priority=13, startTime=1732029234334; duration=0sec 2024-11-19T15:13:54,793 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/A, priority=13, startTime=1732029234333; duration=0sec 2024-11-19T15:13:54,793 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:54,793 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:A 2024-11-19T15:13:54,793 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:54,794 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:54,794 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:B 2024-11-19T15:13:54,795 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:54,795 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/C is initiating minor compaction (all files) 2024-11-19T15:13:54,795 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/C in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:54,795 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/570af6493da04e2bb9d2dc526587cbd5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/a395c1cc3ca6421d9f58efee8f10b3e0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/d865538949e742438fa38c8114796f6f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=36.5 K 2024-11-19T15:13:54,795 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 570af6493da04e2bb9d2dc526587cbd5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732029230001 2024-11-19T15:13:54,796 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting a395c1cc3ca6421d9f58efee8f10b3e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732029231954 2024-11-19T15:13:54,796 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting d865538949e742438fa38c8114796f6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732029233148 2024-11-19T15:13:54,805 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#C#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:54,805 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/4e1a1eb3b9204862a62b55af00837717 is 50, key is test_row_0/C:col10/1732029233148/Put/seqid=0 2024-11-19T15:13:54,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741896_1072 (size=12915) 2024-11-19T15:13:54,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732029234808, max timestamp=1732029234818Num uniques 1145; 2024-11-19T15:13:54,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-19T15:13:54,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732029234808, max timestamp=1732029234818Num uniques 1145; 2024-11-19T15:13:54,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-19T15:13:54,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732029234808, max timestamp=1732029234818Num uniques 1009; 2024-11-19T15:13:54,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11220, suffix heap size=132904,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=245 2024-11-19T15:13:54,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732029234866, max timestamp=1732029234889Num uniques 1104; 2024-11-19T15:13:54,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-19T15:13:54,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732029234866, max timestamp=1732029234889Num uniques 1104; 2024-11-19T15:13:54,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-19T15:13:54,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732029234866, max timestamp=1732029234889Num uniques 1104; 2024-11-19T15:13:54,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-19T15:13:54,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T15:13:54,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:54,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:54,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:54,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:54,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:54,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029294946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:54,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732029234929, max timestamp=1732029234949Num uniques 1113; 2024-11-19T15:13:54,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732029234929, max timestamp=1732029234949Num uniques 1113; 2024-11-19T15:13:54,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-19T15:13:54,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-19T15:13:54,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732029234929, max timestamp=1732029234949Num uniques 1113; 2024-11-19T15:13:54,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-11-19T15:13:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:54,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/14dc77378b424e2ebaed0533becbfba0 2024-11-19T15:13:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/42d3b17fe00348b69eeff51008c21614 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/42d3b17fe00348b69eeff51008c21614 2024-11-19T15:13:54,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/42d3b17fe00348b69eeff51008c21614, entries=250, sequenceid=351, filesize=16.8 K 2024-11-19T15:13:54,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/df547eb2507f4d2d87a398b0b3bd5026 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/df547eb2507f4d2d87a398b0b3bd5026 2024-11-19T15:13:54,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/df547eb2507f4d2d87a398b0b3bd5026, entries=150, sequenceid=351, filesize=12.0 K 2024-11-19T15:13:54,988 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T15:13:54,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/14dc77378b424e2ebaed0533becbfba0 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/14dc77378b424e2ebaed0533becbfba0 2024-11-19T15:13:54,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/14dc77378b424e2ebaed0533becbfba0, entries=150, sequenceid=351, filesize=12.0 K 2024-11-19T15:13:54,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=87.22 KB/89310 for 9a10ee694987b867f252a8f623c69480 in 497ms, sequenceid=351, compaction requested=false 2024-11-19T15:13:54,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:54,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=93.93 KB heapSize=265.08 KB 2024-11-19T15:13:54,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:54,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:54,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:54,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:54,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:54,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:13:55,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/1deeed7441e34bc2a6f1d5d9430d4f9d is 50, key is test_row_0/A:col10/1732029234994/Put/seqid=0 2024-11-19T15:13:55,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741897_1073 (size=17181) 2024-11-19T15:13:55,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/1deeed7441e34bc2a6f1d5d9430d4f9d 2024-11-19T15:13:55,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/2c89aad715ad4771a23a26e199713eb8 is 50, key is test_row_0/B:col10/1732029234994/Put/seqid=0 2024-11-19T15:13:55,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741898_1074 (size=12301) 2024-11-19T15:13:55,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/2c89aad715ad4771a23a26e199713eb8 2024-11-19T15:13:55,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e73ddc039ee244d380f75f418bdcd49e is 50, key is test_row_0/C:col10/1732029234994/Put/seqid=0 2024-11-19T15:13:55,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741899_1075 (size=12301) 2024-11-19T15:13:55,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T15:13:55,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:55,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:55,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:55,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:55,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:55,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029295079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T15:13:55,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029295094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:55,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029295194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:55,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:55,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029295203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:55,222 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/4e1a1eb3b9204862a62b55af00837717 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/4e1a1eb3b9204862a62b55af00837717 2024-11-19T15:13:55,229 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/C of 9a10ee694987b867f252a8f623c69480 into 4e1a1eb3b9204862a62b55af00837717(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:55,229 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:55,230 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/C, priority=13, startTime=1732029234334; duration=0sec 2024-11-19T15:13:55,230 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:55,230 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:C 2024-11-19T15:13:55,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T15:13:55,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:55,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:55,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:55,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:55,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:55,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T15:13:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:55,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:55,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:55,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029295404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:55,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029295414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:55,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e73ddc039ee244d380f75f418bdcd49e 2024-11-19T15:13:55,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/1deeed7441e34bc2a6f1d5d9430d4f9d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1deeed7441e34bc2a6f1d5d9430d4f9d 2024-11-19T15:13:55,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1deeed7441e34bc2a6f1d5d9430d4f9d, entries=250, sequenceid=487, filesize=16.8 K 2024-11-19T15:13:55,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/2c89aad715ad4771a23a26e199713eb8 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/2c89aad715ad4771a23a26e199713eb8 2024-11-19T15:13:55,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/2c89aad715ad4771a23a26e199713eb8, entries=150, sequenceid=487, filesize=12.0 K 2024-11-19T15:13:55,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e73ddc039ee244d380f75f418bdcd49e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e73ddc039ee244d380f75f418bdcd49e 2024-11-19T15:13:55,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e73ddc039ee244d380f75f418bdcd49e, entries=150, sequenceid=487, filesize=12.0 K 2024-11-19T15:13:55,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~265.03 KB/271392, currentSize=87.22 KB/89310 for 9a10ee694987b867f252a8f623c69480 in 509ms, sequenceid=487, compaction requested=true 2024-11-19T15:13:55,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:55,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:13:55,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:55,509 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:55,509 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:55,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:13:55,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:55,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:13:55,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:55,510 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:55,510 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:55,510 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/A is initiating minor compaction (all files) 2024-11-19T15:13:55,510 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/B is initiating minor compaction (all files) 2024-11-19T15:13:55,510 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/A in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:55,510 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/B in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:55,510 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8adc0e9942294697a8a46de6f00dbe0d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/42d3b17fe00348b69eeff51008c21614, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1deeed7441e34bc2a6f1d5d9430d4f9d] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=46.2 K 2024-11-19T15:13:55,510 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/15fa3562efcc45a5b910dbb3e8f86c2b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/df547eb2507f4d2d87a398b0b3bd5026, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/2c89aad715ad4771a23a26e199713eb8] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=36.6 K 2024-11-19T15:13:55,511 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8adc0e9942294697a8a46de6f00dbe0d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732029233148 2024-11-19T15:13:55,511 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 15fa3562efcc45a5b910dbb3e8f86c2b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732029233148 2024-11-19T15:13:55,511 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42d3b17fe00348b69eeff51008c21614, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732029233843 2024-11-19T15:13:55,512 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting df547eb2507f4d2d87a398b0b3bd5026, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732029233862 2024-11-19T15:13:55,512 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1deeed7441e34bc2a6f1d5d9430d4f9d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1732029234962 2024-11-19T15:13:55,512 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c89aad715ad4771a23a26e199713eb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1732029234975 2024-11-19T15:13:55,524 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#A#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:55,525 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/8ddebca81bd343b68e5b3aa85e97499a is 50, key is test_row_0/A:col10/1732029234994/Put/seqid=0 2024-11-19T15:13:55,527 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:55,528 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/3f7a64fa1afa4dc5bc33acdaacf18200 is 50, key is test_row_0/B:col10/1732029234994/Put/seqid=0 2024-11-19T15:13:55,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741900_1076 (size=13017) 2024-11-19T15:13:55,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741901_1077 (size=13017) 2024-11-19T15:13:55,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T15:13:55,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:55,544 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T15:13:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:55,548 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/8ddebca81bd343b68e5b3aa85e97499a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8ddebca81bd343b68e5b3aa85e97499a 2024-11-19T15:13:55,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/6ed564d41b244ef3a26c28cfcb514e52 is 50, key is test_row_0/A:col10/1732029235090/Put/seqid=0 2024-11-19T15:13:55,557 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/A of 9a10ee694987b867f252a8f623c69480 into 8ddebca81bd343b68e5b3aa85e97499a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:55,557 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:55,557 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/A, priority=13, startTime=1732029235508; duration=0sec 2024-11-19T15:13:55,557 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:55,557 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:A 2024-11-19T15:13:55,557 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:55,558 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:55,559 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/C is initiating minor compaction (all files) 2024-11-19T15:13:55,559 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/C in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:55,559 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/4e1a1eb3b9204862a62b55af00837717, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/14dc77378b424e2ebaed0533becbfba0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e73ddc039ee244d380f75f418bdcd49e] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=36.6 K 2024-11-19T15:13:55,560 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e1a1eb3b9204862a62b55af00837717, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732029233148 2024-11-19T15:13:55,560 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14dc77378b424e2ebaed0533becbfba0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732029233862 2024-11-19T15:13:55,561 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e73ddc039ee244d380f75f418bdcd49e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1732029234975 2024-11-19T15:13:55,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741902_1078 (size=12301) 2024-11-19T15:13:55,578 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#C#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:55,579 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/9895c8360cc6465ea0d531d4d948342a is 50, key is test_row_0/C:col10/1732029234994/Put/seqid=0 2024-11-19T15:13:55,580 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/6ed564d41b244ef3a26c28cfcb514e52 2024-11-19T15:13:55,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/0df3e1f282a44bd08b510ecb40aad286 is 50, key is test_row_0/B:col10/1732029235090/Put/seqid=0 2024-11-19T15:13:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T15:13:55,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741903_1079 (size=13017) 2024-11-19T15:13:55,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741904_1080 (size=12301) 2024-11-19T15:13:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:55,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:55,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029295755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:55,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029295763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:55,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029295874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:55,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029295874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:55,944 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/3f7a64fa1afa4dc5bc33acdaacf18200 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/3f7a64fa1afa4dc5bc33acdaacf18200 2024-11-19T15:13:55,951 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/B of 9a10ee694987b867f252a8f623c69480 into 3f7a64fa1afa4dc5bc33acdaacf18200(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:55,951 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:55,951 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/B, priority=13, startTime=1732029235509; duration=0sec 2024-11-19T15:13:55,951 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:55,951 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:B 2024-11-19T15:13:56,013 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/0df3e1f282a44bd08b510ecb40aad286 2024-11-19T15:13:56,014 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/9895c8360cc6465ea0d531d4d948342a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9895c8360cc6465ea0d531d4d948342a 2024-11-19T15:13:56,021 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/C of 9a10ee694987b867f252a8f623c69480 into 9895c8360cc6465ea0d531d4d948342a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:56,021 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:56,021 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/C, priority=13, startTime=1732029235509; duration=0sec 2024-11-19T15:13:56,021 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:56,021 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:C 2024-11-19T15:13:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e2be859882f04f7a9520725f154e8680 is 50, key is test_row_0/C:col10/1732029235090/Put/seqid=0 2024-11-19T15:13:56,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741905_1081 (size=12301) 2024-11-19T15:13:56,031 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e2be859882f04f7a9520725f154e8680 2024-11-19T15:13:56,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/6ed564d41b244ef3a26c28cfcb514e52 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/6ed564d41b244ef3a26c28cfcb514e52 2024-11-19T15:13:56,044 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/6ed564d41b244ef3a26c28cfcb514e52, entries=150, sequenceid=504, filesize=12.0 K 2024-11-19T15:13:56,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/0df3e1f282a44bd08b510ecb40aad286 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/0df3e1f282a44bd08b510ecb40aad286 2024-11-19T15:13:56,053 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/0df3e1f282a44bd08b510ecb40aad286, entries=150, sequenceid=504, filesize=12.0 K 2024-11-19T15:13:56,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e2be859882f04f7a9520725f154e8680 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e2be859882f04f7a9520725f154e8680 2024-11-19T15:13:56,059 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e2be859882f04f7a9520725f154e8680, entries=150, sequenceid=504, filesize=12.0 K 2024-11-19T15:13:56,061 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9a10ee694987b867f252a8f623c69480 in 516ms, sequenceid=504, compaction requested=false 2024-11-19T15:13:56,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:56,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:56,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-19T15:13:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-19T15:13:56,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T15:13:56,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5950 sec 2024-11-19T15:13:56,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6020 sec 2024-11-19T15:13:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:56,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T15:13:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:56,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/718a627e671e47418b0cb2436dcf340f is 50, key is test_row_0/A:col10/1732029236085/Put/seqid=0 2024-11-19T15:13:56,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741906_1082 (size=17181) 2024-11-19T15:13:56,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/718a627e671e47418b0cb2436dcf340f 2024-11-19T15:13:56,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/0bafab0a94f848c28be02bf5cfb1e34a is 50, key is test_row_0/B:col10/1732029236085/Put/seqid=0 2024-11-19T15:13:56,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741907_1083 (size=12301) 2024-11-19T15:13:56,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029296116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:56,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029296119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:56,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029296224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:56,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029296234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:56,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029296434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:56,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029296444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:56,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/0bafab0a94f848c28be02bf5cfb1e34a 2024-11-19T15:13:56,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/7181dcc3211d4383ab98f18e3cf6e297 is 50, key is test_row_0/C:col10/1732029236085/Put/seqid=0 2024-11-19T15:13:56,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741908_1084 (size=12301) 2024-11-19T15:13:56,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/7181dcc3211d4383ab98f18e3cf6e297 2024-11-19T15:13:56,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/718a627e671e47418b0cb2436dcf340f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/718a627e671e47418b0cb2436dcf340f 2024-11-19T15:13:56,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/718a627e671e47418b0cb2436dcf340f, entries=250, sequenceid=525, filesize=16.8 K 2024-11-19T15:13:56,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/0bafab0a94f848c28be02bf5cfb1e34a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/0bafab0a94f848c28be02bf5cfb1e34a 2024-11-19T15:13:56,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/0bafab0a94f848c28be02bf5cfb1e34a, entries=150, sequenceid=525, filesize=12.0 K 2024-11-19T15:13:56,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/7181dcc3211d4383ab98f18e3cf6e297 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/7181dcc3211d4383ab98f18e3cf6e297 2024-11-19T15:13:56,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/7181dcc3211d4383ab98f18e3cf6e297, entries=150, sequenceid=525, filesize=12.0 K 2024-11-19T15:13:56,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9a10ee694987b867f252a8f623c69480 in 481ms, sequenceid=525, compaction requested=true 2024-11-19T15:13:56,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:56,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:13:56,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:56,567 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:56,568 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:13:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:13:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:56,569 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:56,569 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/A is initiating minor compaction (all files) 2024-11-19T15:13:56,569 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/A in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:56,570 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8ddebca81bd343b68e5b3aa85e97499a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/6ed564d41b244ef3a26c28cfcb514e52, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/718a627e671e47418b0cb2436dcf340f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=41.5 K 2024-11-19T15:13:56,570 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:56,570 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/B is initiating minor compaction (all files) 2024-11-19T15:13:56,570 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/B in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:56,570 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/3f7a64fa1afa4dc5bc33acdaacf18200, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/0df3e1f282a44bd08b510ecb40aad286, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/0bafab0a94f848c28be02bf5cfb1e34a] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=36.7 K 2024-11-19T15:13:56,571 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f7a64fa1afa4dc5bc33acdaacf18200, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1732029234975 2024-11-19T15:13:56,571 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ddebca81bd343b68e5b3aa85e97499a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1732029234975 2024-11-19T15:13:56,572 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df3e1f282a44bd08b510ecb40aad286, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732029235068 2024-11-19T15:13:56,572 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ed564d41b244ef3a26c28cfcb514e52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732029235068 2024-11-19T15:13:56,572 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bafab0a94f848c28be02bf5cfb1e34a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1732029235754 2024-11-19T15:13:56,572 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 718a627e671e47418b0cb2436dcf340f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1732029235749 2024-11-19T15:13:56,584 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#B#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:56,585 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#A#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:56,586 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/259f72d143244a72af0520aeabd010a4 is 50, key is test_row_0/A:col10/1732029236085/Put/seqid=0 2024-11-19T15:13:56,587 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/9199c7114cd94482927b7f5ac7574ccc is 50, key is test_row_0/B:col10/1732029236085/Put/seqid=0 2024-11-19T15:13:56,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741910_1086 (size=13119) 2024-11-19T15:13:56,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741909_1085 (size=13119) 2024-11-19T15:13:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T15:13:56,615 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:13:56,615 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:13:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-19T15:13:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T15:13:56,618 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:13:56,619 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:13:56,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:13:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T15:13:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:56,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T15:13:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:56,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/bf53d35890684b7b825a89a3d8aca6a7 is 50, key is test_row_0/A:col10/1732029236112/Put/seqid=0 2024-11-19T15:13:56,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741911_1087 (size=14741) 2024-11-19T15:13:56,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T15:13:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:56,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:56,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:56,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029296820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:56,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029296822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:56,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T15:13:56,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:56,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:56,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:56,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:56,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T15:13:56,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:56,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029296935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029296935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:57,010 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/9199c7114cd94482927b7f5ac7574ccc as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/9199c7114cd94482927b7f5ac7574ccc 2024-11-19T15:13:57,018 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/259f72d143244a72af0520aeabd010a4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/259f72d143244a72af0520aeabd010a4 2024-11-19T15:13:57,026 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/B of 9a10ee694987b867f252a8f623c69480 into 9199c7114cd94482927b7f5ac7574ccc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:57,026 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/A of 9a10ee694987b867f252a8f623c69480 into 259f72d143244a72af0520aeabd010a4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:57,026 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:57,026 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:57,026 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/A, priority=13, startTime=1732029236567; duration=0sec 2024-11-19T15:13:57,026 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/B, priority=13, startTime=1732029236567; duration=0sec 2024-11-19T15:13:57,026 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:13:57,026 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:A 2024-11-19T15:13:57,026 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:57,026 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:B 2024-11-19T15:13:57,026 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:13:57,028 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:13:57,028 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/C is initiating minor compaction (all files) 2024-11-19T15:13:57,028 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/C in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:57,028 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9895c8360cc6465ea0d531d4d948342a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e2be859882f04f7a9520725f154e8680, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/7181dcc3211d4383ab98f18e3cf6e297] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=36.7 K 2024-11-19T15:13:57,029 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9895c8360cc6465ea0d531d4d948342a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1732029234975 2024-11-19T15:13:57,029 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2be859882f04f7a9520725f154e8680, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732029235068 2024-11-19T15:13:57,030 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7181dcc3211d4383ab98f18e3cf6e297, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1732029235754 2024-11-19T15:13:57,041 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#C#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:13:57,043 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/67d85af8ccc746469a7b063c7a440c57 is 50, key is test_row_0/C:col10/1732029236085/Put/seqid=0 2024-11-19T15:13:57,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741912_1088 (size=13119) 2024-11-19T15:13:57,056 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/67d85af8ccc746469a7b063c7a440c57 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/67d85af8ccc746469a7b063c7a440c57 2024-11-19T15:13:57,066 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/C of 9a10ee694987b867f252a8f623c69480 into 67d85af8ccc746469a7b063c7a440c57(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:13:57,066 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:57,066 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/C, priority=13, startTime=1732029236568; duration=0sec 2024-11-19T15:13:57,066 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:13:57,066 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:C 2024-11-19T15:13:57,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T15:13:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:57,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:57,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029297144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:57,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029297144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:57,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/bf53d35890684b7b825a89a3d8aca6a7 2024-11-19T15:13:57,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/56ff621398b94bb686bda314881a3f78 is 50, key is test_row_0/B:col10/1732029236112/Put/seqid=0 2024-11-19T15:13:57,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741913_1089 (size=12301) 2024-11-19T15:13:57,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T15:13:57,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:57,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:57,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:57,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:57,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T15:13:57,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T15:13:57,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:57,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:57,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:57,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:57,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:57,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029297454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:57,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029297454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:57,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T15:13:57,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:57,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:57,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:57,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:57,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:13:57,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/56ff621398b94bb686bda314881a3f78 2024-11-19T15:13:57,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/9d4222a955934a65bd3ba6300d085dc4 is 50, key is test_row_0/C:col10/1732029236112/Put/seqid=0 2024-11-19T15:13:57,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741914_1090 (size=12301) 2024-11-19T15:13:57,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/9d4222a955934a65bd3ba6300d085dc4 2024-11-19T15:13:57,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/bf53d35890684b7b825a89a3d8aca6a7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/bf53d35890684b7b825a89a3d8aca6a7 2024-11-19T15:13:57,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/bf53d35890684b7b825a89a3d8aca6a7, entries=200, sequenceid=540, filesize=14.4 K 2024-11-19T15:13:57,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/56ff621398b94bb686bda314881a3f78 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/56ff621398b94bb686bda314881a3f78 2024-11-19T15:13:57,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/56ff621398b94bb686bda314881a3f78, entries=150, sequenceid=540, filesize=12.0 K 2024-11-19T15:13:57,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/9d4222a955934a65bd3ba6300d085dc4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9d4222a955934a65bd3ba6300d085dc4 2024-11-19T15:13:57,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9d4222a955934a65bd3ba6300d085dc4, entries=150, sequenceid=540, filesize=12.0 K 2024-11-19T15:13:57,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9a10ee694987b867f252a8f623c69480 in 895ms, sequenceid=540, compaction requested=false 2024-11-19T15:13:57,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:57,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T15:13:57,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:57,694 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T15:13:57,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:57,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:57,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:57,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:57,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:57,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:57,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/b731c646230047e584be58259f453566 is 50, key is test_row_0/A:col10/1732029236818/Put/seqid=0 2024-11-19T15:13:57,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741915_1091 (size=12301) 2024-11-19T15:13:57,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T15:13:57,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:57,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:13:57,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:57,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029297993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029297992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:57,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:57,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:57,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:57,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:57,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:57,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:58,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029298104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:58,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:58,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029298104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:58,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:58,110 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/b731c646230047e584be58259f453566 2024-11-19T15:13:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:58,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/1492e5b9ad86428f89aaa720f7d95d94 is 50, key is test_row_0/B:col10/1732029236818/Put/seqid=0 2024-11-19T15:13:58,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741916_1092 (size=12301) 2024-11-19T15:13:58,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57326 deadline: 1732029298234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:58,239 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 9a10ee694987b867f252a8f623c69480 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:13:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:58,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57264 deadline: 1732029298244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:58,247 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in 9a10ee694987b867f252a8f623c69480 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:13:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:58,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57270 deadline: 1732029298255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:58,260 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 9a10ee694987b867f252a8f623c69480 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:13:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:58,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029298314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:58,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029298314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:58,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:58,539 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/1492e5b9ad86428f89aaa720f7d95d94 2024-11-19T15:13:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/2cc81cbb94a6442aac0fcaac57c4b7ce is 50, key is test_row_0/C:col10/1732029236818/Put/seqid=0 2024-11-19T15:13:58,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741917_1093 (size=12301) 2024-11-19T15:13:58,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029298624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:58,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029298624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T15:13:58,958 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/2cc81cbb94a6442aac0fcaac57c4b7ce 2024-11-19T15:13:58,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/b731c646230047e584be58259f453566 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/b731c646230047e584be58259f453566 2024-11-19T15:13:58,970 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/b731c646230047e584be58259f453566, entries=150, sequenceid=560, filesize=12.0 K 2024-11-19T15:13:58,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/1492e5b9ad86428f89aaa720f7d95d94 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/1492e5b9ad86428f89aaa720f7d95d94 2024-11-19T15:13:58,980 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/1492e5b9ad86428f89aaa720f7d95d94, entries=150, sequenceid=560, filesize=12.0 K 2024-11-19T15:13:58,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/2cc81cbb94a6442aac0fcaac57c4b7ce as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2cc81cbb94a6442aac0fcaac57c4b7ce 2024-11-19T15:13:58,987 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2cc81cbb94a6442aac0fcaac57c4b7ce, entries=150, sequenceid=560, filesize=12.0 K 2024-11-19T15:13:58,988 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9a10ee694987b867f252a8f623c69480 in 1294ms, sequenceid=560, compaction requested=true 2024-11-19T15:13:58,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:13:58,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:13:58,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-19T15:13:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-19T15:13:58,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-19T15:13:58,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3720 sec 2024-11-19T15:13:58,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.3790 sec 2024-11-19T15:13:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:13:59,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T15:13:59,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:13:59,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:59,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:13:59,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:59,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:13:59,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:13:59,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/93e918556d50433daf707c499dc242d3 is 50, key is test_row_0/A:col10/1732029239141/Put/seqid=0 2024-11-19T15:13:59,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741918_1094 (size=17181) 2024-11-19T15:13:59,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029299171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:59,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029299173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:59,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029299284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:59,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029299284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:59,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029299494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:59,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029299494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:59,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/93e918556d50433daf707c499dc242d3 2024-11-19T15:13:59,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/da9b866449124e3ab3e4a75c79ae191b is 50, key is test_row_0/B:col10/1732029239141/Put/seqid=0 2024-11-19T15:13:59,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741919_1095 (size=12301) 2024-11-19T15:13:59,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029299804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:59,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:13:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029299804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:13:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:13:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:13:59,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/da9b866449124e3ab3e4a75c79ae191b 2024-11-19T15:13:59,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/c17e7ddf8a7c4a07a9b4edb8c6168cb6 is 50, key is test_row_0/C:col10/1732029239141/Put/seqid=0 2024-11-19T15:14:00,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741920_1096 (size=12301) 2024-11-19T15:14:00,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029300314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:00,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:00,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:00,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:00,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029300314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:00,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/c17e7ddf8a7c4a07a9b4edb8c6168cb6 2024-11-19T15:14:00,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/93e918556d50433daf707c499dc242d3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/93e918556d50433daf707c499dc242d3 2024-11-19T15:14:00,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/93e918556d50433daf707c499dc242d3, entries=250, sequenceid=580, filesize=16.8 K 2024-11-19T15:14:00,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/da9b866449124e3ab3e4a75c79ae191b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/da9b866449124e3ab3e4a75c79ae191b 2024-11-19T15:14:00,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/da9b866449124e3ab3e4a75c79ae191b, entries=150, sequenceid=580, filesize=12.0 K 2024-11-19T15:14:00,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/c17e7ddf8a7c4a07a9b4edb8c6168cb6 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/c17e7ddf8a7c4a07a9b4edb8c6168cb6 2024-11-19T15:14:00,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/c17e7ddf8a7c4a07a9b4edb8c6168cb6, entries=150, sequenceid=580, filesize=12.0 K 2024-11-19T15:14:00,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9a10ee694987b867f252a8f623c69480 in 1285ms, sequenceid=580, compaction requested=true 2024-11-19T15:14:00,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:00,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:00,433 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:14:00,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:00,433 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:14:00,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:00,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:00,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:00,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:00,435 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:14:00,436 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/A is initiating minor compaction (all files) 2024-11-19T15:14:00,436 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/A in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:00,436 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/259f72d143244a72af0520aeabd010a4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/bf53d35890684b7b825a89a3d8aca6a7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/b731c646230047e584be58259f453566, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/93e918556d50433daf707c499dc242d3] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=56.0 K 2024-11-19T15:14:00,436 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:14:00,436 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/B is initiating minor compaction (all files) 2024-11-19T15:14:00,436 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/B in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:00,436 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 259f72d143244a72af0520aeabd010a4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1732029235754 2024-11-19T15:14:00,437 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/9199c7114cd94482927b7f5ac7574ccc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/56ff621398b94bb686bda314881a3f78, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/1492e5b9ad86428f89aaa720f7d95d94, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/da9b866449124e3ab3e4a75c79ae191b] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=48.8 K 2024-11-19T15:14:00,437 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf53d35890684b7b825a89a3d8aca6a7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732029236112 2024-11-19T15:14:00,437 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 9199c7114cd94482927b7f5ac7574ccc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1732029235754 2024-11-19T15:14:00,438 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting b731c646230047e584be58259f453566, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1732029236811 2024-11-19T15:14:00,438 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ff621398b94bb686bda314881a3f78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732029236112 2024-11-19T15:14:00,438 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93e918556d50433daf707c499dc242d3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1732029237985 2024-11-19T15:14:00,439 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 1492e5b9ad86428f89aaa720f7d95d94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1732029236811 2024-11-19T15:14:00,440 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting da9b866449124e3ab3e4a75c79ae191b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1732029239141 2024-11-19T15:14:00,457 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:00,457 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:00,457 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/fb15900e07a94ee4ac7778cc2692ccff is 50, key is test_row_0/B:col10/1732029239141/Put/seqid=0 2024-11-19T15:14:00,458 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/ea747ef0e6ce406c84b24bfc746b7339 is 50, key is test_row_0/A:col10/1732029239141/Put/seqid=0 2024-11-19T15:14:00,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741921_1097 (size=13255) 2024-11-19T15:14:00,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741922_1098 (size=13255) 2024-11-19T15:14:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T15:14:00,785 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:00,786 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-19T15:14:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T15:14:00,790 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:00,791 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:00,791 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:00,889 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/fb15900e07a94ee4ac7778cc2692ccff as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/fb15900e07a94ee4ac7778cc2692ccff 2024-11-19T15:14:00,889 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/ea747ef0e6ce406c84b24bfc746b7339 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/ea747ef0e6ce406c84b24bfc746b7339 2024-11-19T15:14:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T15:14:00,905 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a10ee694987b867f252a8f623c69480/B of 9a10ee694987b867f252a8f623c69480 into fb15900e07a94ee4ac7778cc2692ccff(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:00,905 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:00,905 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/B, priority=12, startTime=1732029240433; duration=0sec 2024-11-19T15:14:00,905 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:00,905 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:B 2024-11-19T15:14:00,905 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:14:00,906 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a10ee694987b867f252a8f623c69480/A of 9a10ee694987b867f252a8f623c69480 into ea747ef0e6ce406c84b24bfc746b7339(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:00,906 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:00,906 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/A, priority=12, startTime=1732029240433; duration=0sec 2024-11-19T15:14:00,906 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:00,907 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:A 2024-11-19T15:14:00,908 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:14:00,908 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/C is initiating minor compaction (all files) 2024-11-19T15:14:00,908 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/C in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:00,908 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/67d85af8ccc746469a7b063c7a440c57, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9d4222a955934a65bd3ba6300d085dc4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2cc81cbb94a6442aac0fcaac57c4b7ce, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/c17e7ddf8a7c4a07a9b4edb8c6168cb6] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=48.8 K 2024-11-19T15:14:00,909 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d85af8ccc746469a7b063c7a440c57, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1732029235754 2024-11-19T15:14:00,909 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d4222a955934a65bd3ba6300d085dc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732029236112 2024-11-19T15:14:00,910 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cc81cbb94a6442aac0fcaac57c4b7ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1732029236811 2024-11-19T15:14:00,911 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting c17e7ddf8a7c4a07a9b4edb8c6168cb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1732029239141 2024-11-19T15:14:00,923 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:00,924 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/6c16990031da4f28a7da66c41ee11827 is 50, key is test_row_0/C:col10/1732029239141/Put/seqid=0 2024-11-19T15:14:00,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741923_1099 (size=13255) 2024-11-19T15:14:00,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T15:14:00,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:00,945 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T15:14:00,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:14:00,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:00,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:14:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:14:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/073bbb945c164b7fb646752ac6c0ec2d is 50, key is test_row_0/A:col10/1732029239169/Put/seqid=0 2024-11-19T15:14:00,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741924_1100 (size=12301) 2024-11-19T15:14:00,962 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/073bbb945c164b7fb646752ac6c0ec2d 2024-11-19T15:14:00,963 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/6c16990031da4f28a7da66c41ee11827 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/6c16990031da4f28a7da66c41ee11827 2024-11-19T15:14:00,974 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a10ee694987b867f252a8f623c69480/C of 9a10ee694987b867f252a8f623c69480 into 6c16990031da4f28a7da66c41ee11827(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:00,975 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:00,975 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/C, priority=12, startTime=1732029240434; duration=0sec 2024-11-19T15:14:00,975 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:00,975 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:C 2024-11-19T15:14:00,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/e11aaf22d52f4cb4a2497766bad5dad5 is 50, key is test_row_0/B:col10/1732029239169/Put/seqid=0 2024-11-19T15:14:00,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741925_1101 (size=12301) 2024-11-19T15:14:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T15:14:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:14:01,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:14:01,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:01,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029301365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:01,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:01,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029301367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:01,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:01,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:01,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:01,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:01,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:01,381 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/e11aaf22d52f4cb4a2497766bad5dad5 2024-11-19T15:14:01,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/2784f57b9a01458baea904afa183c025 is 50, key is test_row_0/C:col10/1732029239169/Put/seqid=0 2024-11-19T15:14:01,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741926_1102 (size=12301) 2024-11-19T15:14:01,405 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/2784f57b9a01458baea904afa183c025 2024-11-19T15:14:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T15:14:01,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/073bbb945c164b7fb646752ac6c0ec2d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/073bbb945c164b7fb646752ac6c0ec2d 2024-11-19T15:14:01,422 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/073bbb945c164b7fb646752ac6c0ec2d, entries=150, sequenceid=594, filesize=12.0 K 2024-11-19T15:14:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/e11aaf22d52f4cb4a2497766bad5dad5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/e11aaf22d52f4cb4a2497766bad5dad5 2024-11-19T15:14:01,430 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/e11aaf22d52f4cb4a2497766bad5dad5, entries=150, sequenceid=594, filesize=12.0 K 2024-11-19T15:14:01,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/2784f57b9a01458baea904afa183c025 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2784f57b9a01458baea904afa183c025 2024-11-19T15:14:01,438 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2784f57b9a01458baea904afa183c025, entries=150, sequenceid=594, filesize=12.0 K 2024-11-19T15:14:01,440 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9a10ee694987b867f252a8f623c69480 in 494ms, sequenceid=594, compaction requested=false 2024-11-19T15:14:01,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:01,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:01,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-19T15:14:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-19T15:14:01,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T15:14:01,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 651 msec 2024-11-19T15:14:01,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 660 msec 2024-11-19T15:14:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:14:01,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029241343, max timestamp=1732029241477Num uniques 859; 2024-11-19T15:14:01,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029241343, max timestamp=1732029241477Num uniques 859; 2024-11-19T15:14:01,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:14:01,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:14:01,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-19T15:14:01,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029241343, max timestamp=1732029241477Num uniques 859; 2024-11-19T15:14:01,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:14:01,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:14:01,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:01,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:14:01,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:01,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:14:01,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:01,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/da1349403ab74037bf68f911ddbf02d5 is 50, key is test_row_0/A:col10/1732029241479/Put/seqid=0 2024-11-19T15:14:01,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741927_1103 (size=14741) 2024-11-19T15:14:01,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732029241520, max timestamp=1732029241533Num uniques 859; 2024-11-19T15:14:01,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732029241520, max timestamp=1732029241533Num uniques 859; 2024-11-19T15:14:01,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T15:14:01,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T15:14:01,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732029241520, max timestamp=1732029241533Num uniques 859; 2024-11-19T15:14:01,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T15:14:01,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732029241573, max timestamp=1732029241616Num uniques 1118; 2024-11-19T15:14:01,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-19T15:14:01,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732029241573, max timestamp=1732029241616Num uniques 1118; 2024-11-19T15:14:01,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-19T15:14:01,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732029241573, max timestamp=1732029241616Num uniques 1118; 2024-11-19T15:14:01,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112  suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-19T15:14:01,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732029241662, max timestamp=1732029241684Num uniques 1127; 2024-11-19T15:14:01,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732029241662, max timestamp=1732029241684Num uniques 1127; 2024-11-19T15:14:01,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-19T15:14:01,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-19T15:14:01,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732029241662, max timestamp=1732029241684Num uniques 1127; 2024-11-19T15:14:01,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-19T15:14:01,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732029241719, max timestamp=1732029241761Num uniques 1086; 2024-11-19T15:14:01,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732029241719, max timestamp=1732029241761Num uniques 1086; 2024-11-19T15:14:01,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732029241719, max timestamp=1732029241761Num uniques 1086; 2024-11-19T15:14:01,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-19T15:14:01,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-19T15:14:01,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-19T15:14:01,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029301834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:01,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:01,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732029241796, max timestamp=1732029241840Num uniques 1145; 2024-11-19T15:14:01,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920  suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-19T15:14:01,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732029241796, max timestamp=1732029241840Num uniques 1145; 2024-11-19T15:14:01,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920  suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-19T15:14:01,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732029241796, max timestamp=1732029241840Num uniques 1145; 2024-11-19T15:14:01,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920  suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-19T15:14:01,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/da1349403ab74037bf68f911ddbf02d5 2024-11-19T15:14:01,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/1bbeef8092a343e79cc30fa420f4443c is 50, key is test_row_0/B:col10/1732029241479/Put/seqid=0 2024-11-19T15:14:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T15:14:01,924 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:01,925 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-19T15:14:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T15:14:01,928 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:01,928 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:01,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:01,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741928_1104 (size=12301) 2024-11-19T15:14:01,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/1bbeef8092a343e79cc30fa420f4443c 2024-11-19T15:14:01,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029301949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:01,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/97a5472d2d3e498d8e0d9156803d45e9 is 50, key is test_row_0/C:col10/1732029241479/Put/seqid=0 2024-11-19T15:14:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:01,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029301954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:01,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741929_1105 (size=12301) 2024-11-19T15:14:01,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/97a5472d2d3e498d8e0d9156803d45e9 2024-11-19T15:14:01,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/da1349403ab74037bf68f911ddbf02d5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/da1349403ab74037bf68f911ddbf02d5 2024-11-19T15:14:01,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/da1349403ab74037bf68f911ddbf02d5, entries=200, sequenceid=617, filesize=14.4 K 2024-11-19T15:14:01,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/1bbeef8092a343e79cc30fa420f4443c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/1bbeef8092a343e79cc30fa420f4443c 2024-11-19T15:14:01,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/1bbeef8092a343e79cc30fa420f4443c, entries=150, sequenceid=617, filesize=12.0 K 2024-11-19T15:14:01,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/97a5472d2d3e498d8e0d9156803d45e9 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/97a5472d2d3e498d8e0d9156803d45e9 2024-11-19T15:14:01,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/97a5472d2d3e498d8e0d9156803d45e9, entries=150, sequenceid=617, filesize=12.0 K 2024-11-19T15:14:01,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 9a10ee694987b867f252a8f623c69480 in 518ms, sequenceid=617, compaction requested=true 2024-11-19T15:14:01,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:01,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:01,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:01,999 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:01,999 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:02,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:02,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:02,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:02,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:02,005 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:02,005 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/B is initiating minor compaction (all files) 2024-11-19T15:14:02,005 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/B in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:02,005 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/fb15900e07a94ee4ac7778cc2692ccff, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/e11aaf22d52f4cb4a2497766bad5dad5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/1bbeef8092a343e79cc30fa420f4443c] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=37.0 K 2024-11-19T15:14:02,006 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:02,006 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/A is initiating minor compaction (all files) 2024-11-19T15:14:02,006 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/A in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:02,006 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/ea747ef0e6ce406c84b24bfc746b7339, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/073bbb945c164b7fb646752ac6c0ec2d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/da1349403ab74037bf68f911ddbf02d5] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=39.4 K 2024-11-19T15:14:02,006 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting fb15900e07a94ee4ac7778cc2692ccff, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1732029239141 2024-11-19T15:14:02,007 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea747ef0e6ce406c84b24bfc746b7339, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1732029239141 2024-11-19T15:14:02,007 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting e11aaf22d52f4cb4a2497766bad5dad5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1732029239167 2024-11-19T15:14:02,008 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 073bbb945c164b7fb646752ac6c0ec2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1732029239167 2024-11-19T15:14:02,008 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bbeef8092a343e79cc30fa420f4443c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1732029241343 2024-11-19T15:14:02,008 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting da1349403ab74037bf68f911ddbf02d5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1732029241343 2024-11-19T15:14:02,019 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:02,020 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/b44866a3d4f5492293f19cbe1d2d1bbb is 50, key is test_row_0/B:col10/1732029241479/Put/seqid=0 2024-11-19T15:14:02,027 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#A#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:02,028 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/fc1e922c003b423097c0fd4c43890be6 is 50, key is test_row_0/A:col10/1732029241479/Put/seqid=0 2024-11-19T15:14:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T15:14:02,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741930_1106 (size=13357) 2024-11-19T15:14:02,053 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/b44866a3d4f5492293f19cbe1d2d1bbb as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/b44866a3d4f5492293f19cbe1d2d1bbb 2024-11-19T15:14:02,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741931_1107 (size=13357) 2024-11-19T15:14:02,064 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/B of 9a10ee694987b867f252a8f623c69480 into b44866a3d4f5492293f19cbe1d2d1bbb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:02,064 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:02,064 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/B, priority=13, startTime=1732029241999; duration=0sec 2024-11-19T15:14:02,064 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:02,065 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:B 2024-11-19T15:14:02,065 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:02,066 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:02,066 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/C is initiating minor compaction (all files) 2024-11-19T15:14:02,066 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/C in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:02,067 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/6c16990031da4f28a7da66c41ee11827, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2784f57b9a01458baea904afa183c025, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/97a5472d2d3e498d8e0d9156803d45e9] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=37.0 K 2024-11-19T15:14:02,067 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c16990031da4f28a7da66c41ee11827, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1732029239141 2024-11-19T15:14:02,069 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 2784f57b9a01458baea904afa183c025, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1732029239167 2024-11-19T15:14:02,069 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 97a5472d2d3e498d8e0d9156803d45e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1732029241343 2024-11-19T15:14:02,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T15:14:02,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:02,083 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-11-19T15:14:02,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:14:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T15:14:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:14:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T15:14:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:14:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T15:14:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:14:02,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:14:02,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/dc03c64df3e74698884ac5b3c1ce22d1 is 50, key is test_row_0/A:col10/1732029241939/Put/seqid=0 2024-11-19T15:14:02,103 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#C#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:02,103 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/1ed68937360041088ab96c59c9b481dd is 50, key is test_row_0/C:col10/1732029241479/Put/seqid=0 2024-11-19T15:14:02,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741932_1108 (size=14741) 2024-11-19T15:14:02,114 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=725 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/dc03c64df3e74698884ac5b3c1ce22d1 2024-11-19T15:14:02,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029302116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:02,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:02,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/6672dd9dbb2846b3b41071f7eb955442 is 50, key is test_row_0/B:col10/1732029241939/Put/seqid=0 2024-11-19T15:14:02,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741933_1109 (size=13357) 2024-11-19T15:14:02,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029302165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:02,167 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/1ed68937360041088ab96c59c9b481dd as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/1ed68937360041088ab96c59c9b481dd 2024-11-19T15:14:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:02,176 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/C of 9a10ee694987b867f252a8f623c69480 into 1ed68937360041088ab96c59c9b481dd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:02,176 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:02,176 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/C, priority=13, startTime=1732029242000; duration=0sec 2024-11-19T15:14:02,177 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:02,177 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:C 2024-11-19T15:14:02,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741934_1110 (size=12301) 2024-11-19T15:14:02,203 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=725 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/6672dd9dbb2846b3b41071f7eb955442 2024-11-19T15:14:02,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/942119cf30da43a6920f61199e0dc1fc is 50, key is test_row_0/C:col10/1732029241939/Put/seqid=0 2024-11-19T15:14:02,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029302235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:02,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T15:14:02,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741935_1111 (size=12301) 2024-11-19T15:14:02,253 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=725 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/942119cf30da43a6920f61199e0dc1fc 2024-11-19T15:14:02,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/dc03c64df3e74698884ac5b3c1ce22d1 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/dc03c64df3e74698884ac5b3c1ce22d1 2024-11-19T15:14:02,280 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/dc03c64df3e74698884ac5b3c1ce22d1, entries=200, sequenceid=725, filesize=14.4 K 2024-11-19T15:14:02,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/6672dd9dbb2846b3b41071f7eb955442 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/6672dd9dbb2846b3b41071f7eb955442 2024-11-19T15:14:02,290 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/6672dd9dbb2846b3b41071f7eb955442, entries=150, sequenceid=725, filesize=12.0 K 2024-11-19T15:14:02,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/942119cf30da43a6920f61199e0dc1fc as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/942119cf30da43a6920f61199e0dc1fc 2024-11-19T15:14:02,305 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/942119cf30da43a6920f61199e0dc1fc, entries=150, sequenceid=725, filesize=12.0 K 2024-11-19T15:14:02,307 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~452.58 KB/463440, currentSize=20.13 KB/20610 for 9a10ee694987b867f252a8f623c69480 in 225ms, sequenceid=725, compaction requested=false 2024-11-19T15:14:02,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:02,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:02,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-19T15:14:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-19T15:14:02,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-19T15:14:02,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 382 msec 2024-11-19T15:14:02,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 394 msec 2024-11-19T15:14:02,467 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/fc1e922c003b423097c0fd4c43890be6 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/fc1e922c003b423097c0fd4c43890be6 2024-11-19T15:14:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:14:02,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:14:02,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:14:02,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:02,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:14:02,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:02,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:14:02,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:02,476 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/A of 9a10ee694987b867f252a8f623c69480 into fc1e922c003b423097c0fd4c43890be6(size=13.0 K), total size for store is 27.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:02,476 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:02,477 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/A, priority=13, startTime=1732029241999; duration=0sec 2024-11-19T15:14:02,477 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:02,477 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:A 2024-11-19T15:14:02,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/a5b7d2c055a14572a7e714afa6896f34 is 50, key is test_row_0/A:col10/1732029242463/Put/seqid=0 2024-11-19T15:14:02,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741936_1112 (size=14737) 2024-11-19T15:14:02,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=736 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/a5b7d2c055a14572a7e714afa6896f34 2024-11-19T15:14:02,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/f5c440fef5fa4b368b60b33b5f2ef070 is 50, key is test_row_0/B:col10/1732029242463/Put/seqid=0 2024-11-19T15:14:02,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029242519, max timestamp=1732029242536Num uniques 859; 2024-11-19T15:14:02,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:02,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029242519, max timestamp=1732029242536Num uniques 859; 2024-11-19T15:14:02,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:02,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029242519, max timestamp=1732029242536Num uniques 859; 2024-11-19T15:14:02,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T15:14:02,555 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:02,556 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-19T15:14:02,558 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:02,560 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:02,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:02,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T15:14:02,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741937_1113 (size=9857) 2024-11-19T15:14:02,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=736 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/f5c440fef5fa4b368b60b33b5f2ef070 2024-11-19T15:14:02,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/5260887ab0f24f35b721bbf1c1153423 is 50, key is test_row_0/C:col10/1732029242463/Put/seqid=0 2024-11-19T15:14:02,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741938_1114 (size=9857) 2024-11-19T15:14:02,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029302610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:02,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:02,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029302614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T15:14:02,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T15:14:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:14:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:02,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:02,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:02,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029302731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:02,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029302734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:02,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T15:14:02,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T15:14:02,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:02,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:14:02,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:02,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:02,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:02,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029302944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:02,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029302944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:02,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:02,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:03,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=736 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/5260887ab0f24f35b721bbf1c1153423 2024-11-19T15:14:03,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/a5b7d2c055a14572a7e714afa6896f34 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/a5b7d2c055a14572a7e714afa6896f34 2024-11-19T15:14:03,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/a5b7d2c055a14572a7e714afa6896f34, entries=200, sequenceid=736, filesize=14.4 K 2024-11-19T15:14:03,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/f5c440fef5fa4b368b60b33b5f2ef070 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/f5c440fef5fa4b368b60b33b5f2ef070 2024-11-19T15:14:03,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T15:14:03,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:03,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:14:03,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:03,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:03,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:03,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/f5c440fef5fa4b368b60b33b5f2ef070, entries=100, sequenceid=736, filesize=9.6 K 2024-11-19T15:14:03,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/5260887ab0f24f35b721bbf1c1153423 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5260887ab0f24f35b721bbf1c1153423 2024-11-19T15:14:03,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5260887ab0f24f35b721bbf1c1153423, entries=100, sequenceid=736, filesize=9.6 K 2024-11-19T15:14:03,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9a10ee694987b867f252a8f623c69480 in 589ms, sequenceid=736, compaction requested=true 2024-11-19T15:14:03,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:03,060 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:03,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:03,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:03,060 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:03,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:03,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:03,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:03,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:03,061 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:03,061 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/A is initiating minor compaction (all files) 2024-11-19T15:14:03,061 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/A in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:03,062 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/fc1e922c003b423097c0fd4c43890be6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/dc03c64df3e74698884ac5b3c1ce22d1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/a5b7d2c055a14572a7e714afa6896f34] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=41.8 K 2024-11-19T15:14:03,062 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc1e922c003b423097c0fd4c43890be6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1732029241343 2024-11-19T15:14:03,063 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:03,063 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc03c64df3e74698884ac5b3c1ce22d1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=725, earliestPutTs=1732029241925 2024-11-19T15:14:03,063 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/B is initiating minor compaction (all files) 2024-11-19T15:14:03,063 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/B in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:03,063 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/b44866a3d4f5492293f19cbe1d2d1bbb, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/6672dd9dbb2846b3b41071f7eb955442, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/f5c440fef5fa4b368b60b33b5f2ef070] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=34.7 K 2024-11-19T15:14:03,063 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5b7d2c055a14572a7e714afa6896f34, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=736, earliestPutTs=1732029242448 2024-11-19T15:14:03,064 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting b44866a3d4f5492293f19cbe1d2d1bbb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1732029241343 2024-11-19T15:14:03,065 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 6672dd9dbb2846b3b41071f7eb955442, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=725, earliestPutTs=1732029241939 2024-11-19T15:14:03,065 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting f5c440fef5fa4b368b60b33b5f2ef070, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=736, earliestPutTs=1732029242463 2024-11-19T15:14:03,083 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:03,083 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:03,084 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/7d0850d9fc094553a12294c1e9bdd31c is 50, key is test_row_0/A:col10/1732029242463/Put/seqid=0 2024-11-19T15:14:03,084 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/47ef636fe42e4a76a16b136062f637e5 is 50, key is test_row_0/B:col10/1732029242463/Put/seqid=0 2024-11-19T15:14:03,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741940_1116 (size=13459) 2024-11-19T15:14:03,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741939_1115 (size=13459) 2024-11-19T15:14:03,114 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/7d0850d9fc094553a12294c1e9bdd31c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/7d0850d9fc094553a12294c1e9bdd31c 2024-11-19T15:14:03,121 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/47ef636fe42e4a76a16b136062f637e5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/47ef636fe42e4a76a16b136062f637e5 2024-11-19T15:14:03,122 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/A of 9a10ee694987b867f252a8f623c69480 into 7d0850d9fc094553a12294c1e9bdd31c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:03,122 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:03,122 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/A, priority=13, startTime=1732029243060; duration=0sec 2024-11-19T15:14:03,123 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:03,123 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:A 2024-11-19T15:14:03,123 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:03,125 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:03,125 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/C is initiating minor compaction (all files) 2024-11-19T15:14:03,125 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/C in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:03,125 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/1ed68937360041088ab96c59c9b481dd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/942119cf30da43a6920f61199e0dc1fc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5260887ab0f24f35b721bbf1c1153423] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=34.7 K 2024-11-19T15:14:03,126 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ed68937360041088ab96c59c9b481dd, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1732029241343 2024-11-19T15:14:03,127 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 942119cf30da43a6920f61199e0dc1fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=725, earliestPutTs=1732029241939 2024-11-19T15:14:03,128 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5260887ab0f24f35b721bbf1c1153423, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=736, earliestPutTs=1732029242463 2024-11-19T15:14:03,128 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/B of 9a10ee694987b867f252a8f623c69480 into 47ef636fe42e4a76a16b136062f637e5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:03,128 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:03,128 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/B, priority=13, startTime=1732029243060; duration=0sec 2024-11-19T15:14:03,128 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:03,128 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:B 2024-11-19T15:14:03,144 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#C#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:03,145 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/d59cca8734fb46abbddf2e05fad9cf3e is 50, key is test_row_0/C:col10/1732029242463/Put/seqid=0 2024-11-19T15:14:03,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741941_1117 (size=13459) 2024-11-19T15:14:03,172 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/d59cca8734fb46abbddf2e05fad9cf3e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/d59cca8734fb46abbddf2e05fad9cf3e 2024-11-19T15:14:03,182 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/C of 9a10ee694987b867f252a8f623c69480 into d59cca8734fb46abbddf2e05fad9cf3e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:03,182 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:03,182 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/C, priority=13, startTime=1732029243061; duration=0sec 2024-11-19T15:14:03,182 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:03,182 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:C 2024-11-19T15:14:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T15:14:03,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T15:14:03,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:03,193 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T15:14:03,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:14:03,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:03,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:14:03,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:03,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:14:03,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:03,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/1dff0fa4c8554443b4d6206ff1ba098f is 50, key is test_row_0/A:col10/1732029242566/Put/seqid=0 2024-11-19T15:14:03,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741942_1118 (size=12301) 2024-11-19T15:14:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:14:03,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. as already flushing 2024-11-19T15:14:03,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 331 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029303270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:03,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 339 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029303273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:03,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029303384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 340 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029303384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:03,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029303594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:03,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 341 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029303594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:03,615 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=776 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/1dff0fa4c8554443b4d6206ff1ba098f 2024-11-19T15:14:03,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/94ae798e13e7493ea543fae21868a6c8 is 50, key is test_row_0/B:col10/1732029242566/Put/seqid=0 2024-11-19T15:14:03,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741943_1119 (size=12301) 2024-11-19T15:14:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T15:14:03,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57302 deadline: 1732029303914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:03,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 342 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57328 deadline: 1732029303914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:03,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:03,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:04,035 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=776 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/94ae798e13e7493ea543fae21868a6c8 2024-11-19T15:14:04,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/7adba33f041e4217bc1605b0f381d13a is 50, key is test_row_0/C:col10/1732029242566/Put/seqid=0 2024-11-19T15:14:04,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741944_1120 (size=12301) 2024-11-19T15:14:04,051 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=776 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/7adba33f041e4217bc1605b0f381d13a 2024-11-19T15:14:04,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/1dff0fa4c8554443b4d6206ff1ba098f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1dff0fa4c8554443b4d6206ff1ba098f 2024-11-19T15:14:04,062 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1dff0fa4c8554443b4d6206ff1ba098f, entries=150, sequenceid=776, filesize=12.0 K 2024-11-19T15:14:04,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/94ae798e13e7493ea543fae21868a6c8 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/94ae798e13e7493ea543fae21868a6c8 2024-11-19T15:14:04,069 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/94ae798e13e7493ea543fae21868a6c8, entries=150, sequenceid=776, filesize=12.0 K 2024-11-19T15:14:04,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/7adba33f041e4217bc1605b0f381d13a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/7adba33f041e4217bc1605b0f381d13a 2024-11-19T15:14:04,075 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/7adba33f041e4217bc1605b0f381d13a, entries=150, sequenceid=776, filesize=12.0 K 2024-11-19T15:14:04,076 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 9a10ee694987b867f252a8f623c69480 in 883ms, sequenceid=776, compaction requested=false 2024-11-19T15:14:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-19T15:14:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-19T15:14:04,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-19T15:14:04,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5170 sec 2024-11-19T15:14:04,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.5240 sec 2024-11-19T15:14:04,233 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-19T15:14:04,233 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-19T15:14:04,233 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:04,233 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:04,239 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:04,239 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:04,240 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-19T15:14:04,240 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:04,240 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:04,240 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:04,240 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:04,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:04,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:04,240 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:04,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:04,241 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-19T15:14:04,241 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:04,241 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:04,241 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:04,241 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9a10ee694987b867f252a8f623c69480 2024-11-19T15:14:04,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T15:14:04,426 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-19T15:14:04,426 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-19T15:14:04,426 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:04,426 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:04,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:14:04,426 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:04,426 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:04,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:04,426 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:04,426 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:04,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:14:04,426 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:04,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:04,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:14:04,426 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:04,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:04,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/5ada67c2162f403db4fc0a55beb0c0b7 is 50, key is test_row_0/A:col10/1732029244424/Put/seqid=0 2024-11-19T15:14:04,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741945_1121 (size=12301) 2024-11-19T15:14:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T15:14:04,714 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:04,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=788 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/5ada67c2162f403db4fc0a55beb0c0b7 2024-11-19T15:14:04,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/436bb2a9f4cf4e179dac4381585af41f is 50, key is test_row_0/B:col10/1732029244424/Put/seqid=0 2024-11-19T15:14:04,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741946_1122 (size=12301) 2024-11-19T15:14:05,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=788 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/436bb2a9f4cf4e179dac4381585af41f 2024-11-19T15:14:05,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/9671e72e093c4a3bb8d8e931634aaa8f is 50, key is test_row_0/C:col10/1732029244424/Put/seqid=0 2024-11-19T15:14:05,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741947_1123 (size=12301) 2024-11-19T15:14:05,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=788 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/9671e72e093c4a3bb8d8e931634aaa8f 2024-11-19T15:14:05,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/5ada67c2162f403db4fc0a55beb0c0b7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/5ada67c2162f403db4fc0a55beb0c0b7 2024-11-19T15:14:05,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/5ada67c2162f403db4fc0a55beb0c0b7, entries=150, sequenceid=788, filesize=12.0 K 2024-11-19T15:14:05,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/436bb2a9f4cf4e179dac4381585af41f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/436bb2a9f4cf4e179dac4381585af41f 2024-11-19T15:14:05,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/436bb2a9f4cf4e179dac4381585af41f, entries=150, sequenceid=788, filesize=12.0 K 2024-11-19T15:14:05,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/9671e72e093c4a3bb8d8e931634aaa8f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9671e72e093c4a3bb8d8e931634aaa8f 2024-11-19T15:14:05,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9671e72e093c4a3bb8d8e931634aaa8f, entries=150, sequenceid=788, filesize=12.0 K 2024-11-19T15:14:05,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 9a10ee694987b867f252a8f623c69480 in 1254ms, sequenceid=788, compaction requested=true 2024-11-19T15:14:05,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:05,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:05,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:05,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:05,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:05,681 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:05,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a10ee694987b867f252a8f623c69480:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:05,681 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:05,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:05,682 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:05,682 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/A is initiating minor compaction (all files) 2024-11-19T15:14:05,682 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:05,682 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/A in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:05,682 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/B is initiating minor compaction (all files) 2024-11-19T15:14:05,682 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/7d0850d9fc094553a12294c1e9bdd31c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1dff0fa4c8554443b4d6206ff1ba098f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/5ada67c2162f403db4fc0a55beb0c0b7] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=37.2 K 2024-11-19T15:14:05,682 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/B in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:05,682 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/47ef636fe42e4a76a16b136062f637e5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/94ae798e13e7493ea543fae21868a6c8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/436bb2a9f4cf4e179dac4381585af41f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=37.2 K 2024-11-19T15:14:05,682 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d0850d9fc094553a12294c1e9bdd31c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=736, earliestPutTs=1732029242072 2024-11-19T15:14:05,682 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 47ef636fe42e4a76a16b136062f637e5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=736, earliestPutTs=1732029242072 2024-11-19T15:14:05,682 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dff0fa4c8554443b4d6206ff1ba098f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=776, earliestPutTs=1732029242566 2024-11-19T15:14:05,683 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ada67c2162f403db4fc0a55beb0c0b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732029243271 2024-11-19T15:14:05,683 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ae798e13e7493ea543fae21868a6c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=776, earliestPutTs=1732029242566 2024-11-19T15:14:05,683 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 436bb2a9f4cf4e179dac4381585af41f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732029243271 2024-11-19T15:14:05,690 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#B#compaction#112 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:05,691 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#A#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:05,691 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/de51146be00f44abb6ff5f60e0a14edf is 50, key is test_row_0/B:col10/1732029244424/Put/seqid=0 2024-11-19T15:14:05,692 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/04f55619a7a74c2aa8544e9637009dd7 is 50, key is test_row_0/A:col10/1732029244424/Put/seqid=0 2024-11-19T15:14:05,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741948_1124 (size=13561) 2024-11-19T15:14:05,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741949_1125 (size=13561) 2024-11-19T15:14:06,101 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/04f55619a7a74c2aa8544e9637009dd7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/04f55619a7a74c2aa8544e9637009dd7 2024-11-19T15:14:06,101 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/de51146be00f44abb6ff5f60e0a14edf as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/de51146be00f44abb6ff5f60e0a14edf 2024-11-19T15:14:06,107 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/A of 9a10ee694987b867f252a8f623c69480 into 04f55619a7a74c2aa8544e9637009dd7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:06,107 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/B of 9a10ee694987b867f252a8f623c69480 into de51146be00f44abb6ff5f60e0a14edf(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:06,107 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:06,107 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:06,107 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/A, priority=13, startTime=1732029245680; duration=0sec 2024-11-19T15:14:06,107 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/B, priority=13, startTime=1732029245681; duration=0sec 2024-11-19T15:14:06,107 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:06,107 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:A 2024-11-19T15:14:06,107 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:06,107 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:06,107 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:B 2024-11-19T15:14:06,108 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:06,109 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9a10ee694987b867f252a8f623c69480/C is initiating minor compaction (all files) 2024-11-19T15:14:06,109 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a10ee694987b867f252a8f623c69480/C in TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:06,109 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/d59cca8734fb46abbddf2e05fad9cf3e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/7adba33f041e4217bc1605b0f381d13a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9671e72e093c4a3bb8d8e931634aaa8f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp, totalSize=37.2 K 2024-11-19T15:14:06,109 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting d59cca8734fb46abbddf2e05fad9cf3e, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=736, earliestPutTs=1732029242072 2024-11-19T15:14:06,110 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7adba33f041e4217bc1605b0f381d13a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=776, earliestPutTs=1732029242566 2024-11-19T15:14:06,110 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9671e72e093c4a3bb8d8e931634aaa8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732029243271 2024-11-19T15:14:06,119 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a10ee694987b867f252a8f623c69480#C#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:06,119 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e49af6e75f2641e0b722dfaa7a705d12 is 50, key is test_row_0/C:col10/1732029244424/Put/seqid=0 2024-11-19T15:14:06,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741950_1126 (size=13561) 2024-11-19T15:14:06,146 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/e49af6e75f2641e0b722dfaa7a705d12 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e49af6e75f2641e0b722dfaa7a705d12 2024-11-19T15:14:06,152 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a10ee694987b867f252a8f623c69480/C of 9a10ee694987b867f252a8f623c69480 into e49af6e75f2641e0b722dfaa7a705d12(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:06,152 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a10ee694987b867f252a8f623c69480: 2024-11-19T15:14:06,152 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., storeName=9a10ee694987b867f252a8f623c69480/C, priority=13, startTime=1732029245681; duration=0sec 2024-11-19T15:14:06,152 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:06,152 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a10ee694987b867f252a8f623c69480:C 2024-11-19T15:14:06,979 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T15:14:08,296 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-19T15:14:08,296 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:08,296 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:08,296 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:08,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:08,306 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-19T15:14:08,306 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:08,306 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:08,306 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:08,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:08,336 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-19T15:14:08,336 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:08,337 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:08,337 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:08,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:08,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T15:14:08,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 275 2024-11-19T15:14:08,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-19T15:14:08,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 268 2024-11-19T15:14:08,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-19T15:14:08,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-19T15:14:08,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T15:14:08,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2521 2024-11-19T15:14:08,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2519 2024-11-19T15:14:08,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T15:14:08,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1143 2024-11-19T15:14:08,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3428 rows 2024-11-19T15:14:08,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1149 2024-11-19T15:14:08,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3437 rows 2024-11-19T15:14:08,338 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T15:14:08,338 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T15:14:08,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:08,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:08,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:08,343 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T15:14:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T15:14:08,354 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029248354"}]},"ts":"1732029248354"} 2024-11-19T15:14:08,357 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T15:14:08,357 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T15:14:08,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T15:14:08,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a10ee694987b867f252a8f623c69480, UNASSIGN}] 2024-11-19T15:14:08,366 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a10ee694987b867f252a8f623c69480, UNASSIGN 2024-11-19T15:14:08,368 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9a10ee694987b867f252a8f623c69480, regionState=CLOSING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:08,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a10ee694987b867f252a8f623c69480, UNASSIGN because future has completed 2024-11-19T15:14:08,372 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T15:14:08,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:14:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T15:14:08,531 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 9a10ee694987b867f252a8f623c69480 2024-11-19T15:14:08,532 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T15:14:08,532 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 9a10ee694987b867f252a8f623c69480, disabling compactions & flushes 2024-11-19T15:14:08,533 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:08,533 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:08,533 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. after waiting 0 ms 2024-11-19T15:14:08,533 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:08,533 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 9a10ee694987b867f252a8f623c69480 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-19T15:14:08,533 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=A 2024-11-19T15:14:08,533 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:08,533 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=B 2024-11-19T15:14:08,534 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:08,534 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9a10ee694987b867f252a8f623c69480, store=C 2024-11-19T15:14:08,534 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:08,540 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/cb854e4161004183ab47ffb7a5f6af9c is 50, key is test_row_1/A:col10/1732029248305/Put/seqid=0 2024-11-19T15:14:08,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741951_1127 (size=9857) 2024-11-19T15:14:08,551 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=797 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/cb854e4161004183ab47ffb7a5f6af9c 2024-11-19T15:14:08,564 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/5f2334d69ebd45b7848ef2fbe74bcfa7 is 50, key is test_row_1/B:col10/1732029248305/Put/seqid=0 2024-11-19T15:14:08,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741952_1128 (size=9857) 2024-11-19T15:14:08,572 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=797 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/5f2334d69ebd45b7848ef2fbe74bcfa7 2024-11-19T15:14:08,583 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/10c23fd7b5ed4ba282f64332b0200d68 is 50, key is test_row_1/C:col10/1732029248305/Put/seqid=0 2024-11-19T15:14:08,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741953_1129 (size=9857) 2024-11-19T15:14:08,589 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=797 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/10c23fd7b5ed4ba282f64332b0200d68 2024-11-19T15:14:08,596 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/A/cb854e4161004183ab47ffb7a5f6af9c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/cb854e4161004183ab47ffb7a5f6af9c 2024-11-19T15:14:08,601 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/cb854e4161004183ab47ffb7a5f6af9c, entries=100, sequenceid=797, filesize=9.6 K 2024-11-19T15:14:08,602 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/B/5f2334d69ebd45b7848ef2fbe74bcfa7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/5f2334d69ebd45b7848ef2fbe74bcfa7 2024-11-19T15:14:08,608 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/5f2334d69ebd45b7848ef2fbe74bcfa7, entries=100, sequenceid=797, filesize=9.6 K 2024-11-19T15:14:08,612 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/.tmp/C/10c23fd7b5ed4ba282f64332b0200d68 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/10c23fd7b5ed4ba282f64332b0200d68 2024-11-19T15:14:08,618 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/10c23fd7b5ed4ba282f64332b0200d68, entries=100, sequenceid=797, filesize=9.6 K 2024-11-19T15:14:08,619 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 9a10ee694987b867f252a8f623c69480 in 86ms, sequenceid=797, compaction requested=false 2024-11-19T15:14:08,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/2b534ab50dba4556987f571fe0d28a45, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/c33944b29bdc40b49b4c01fe9681f9ad, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/0ee9d2d9b4f649bfa728b8c49c5aa6cd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1553e0ce38d14fa681b2191bf8d90545, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/0a508510351c4aa6a735244820bb7696, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/b71df9daf2754ad68695416ed52532f2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/cb898b3c091c4706969eb1e4dc5fbcde, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/5d4c7ab4d08d455fa9c5b14bf03a207e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/e66ab7a7b75c473498dfc6d9522137b6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/6a5b02a22a4e49f7b4ae7fdfbca4f249, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/60798c3f1ddc40d3bfec465e1334ba6b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/30db31b174cc4f199ac4d1181241274a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/eb628ca0f5e744e9abe685be08d214e1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/935d72b396ef4e64a3770b31e4d22d8f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/d817fda905154e67863a8196521a13e4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8010018295b24ac9bde6392f49bf2587, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/4536a3e533f0437e99ff7d7999df9c37, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/94becc5056ee43dd84ddf486b0bc83fe, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8adc0e9942294697a8a46de6f00dbe0d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/42d3b17fe00348b69eeff51008c21614, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1deeed7441e34bc2a6f1d5d9430d4f9d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8ddebca81bd343b68e5b3aa85e97499a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/6ed564d41b244ef3a26c28cfcb514e52, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/718a627e671e47418b0cb2436dcf340f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/259f72d143244a72af0520aeabd010a4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/bf53d35890684b7b825a89a3d8aca6a7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/b731c646230047e584be58259f453566, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/93e918556d50433daf707c499dc242d3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/ea747ef0e6ce406c84b24bfc746b7339, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/073bbb945c164b7fb646752ac6c0ec2d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/da1349403ab74037bf68f911ddbf02d5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/fc1e922c003b423097c0fd4c43890be6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/dc03c64df3e74698884ac5b3c1ce22d1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/a5b7d2c055a14572a7e714afa6896f34, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/7d0850d9fc094553a12294c1e9bdd31c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1dff0fa4c8554443b4d6206ff1ba098f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/5ada67c2162f403db4fc0a55beb0c0b7] to archive 2024-11-19T15:14:08,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:14:08,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/2b534ab50dba4556987f571fe0d28a45 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/2b534ab50dba4556987f571fe0d28a45 2024-11-19T15:14:08,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/c33944b29bdc40b49b4c01fe9681f9ad to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/c33944b29bdc40b49b4c01fe9681f9ad 2024-11-19T15:14:08,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/0ee9d2d9b4f649bfa728b8c49c5aa6cd to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/0ee9d2d9b4f649bfa728b8c49c5aa6cd 2024-11-19T15:14:08,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1553e0ce38d14fa681b2191bf8d90545 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1553e0ce38d14fa681b2191bf8d90545 2024-11-19T15:14:08,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/0a508510351c4aa6a735244820bb7696 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/0a508510351c4aa6a735244820bb7696 2024-11-19T15:14:08,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/b71df9daf2754ad68695416ed52532f2 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/b71df9daf2754ad68695416ed52532f2 2024-11-19T15:14:08,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/cb898b3c091c4706969eb1e4dc5fbcde to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/cb898b3c091c4706969eb1e4dc5fbcde 2024-11-19T15:14:08,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/5d4c7ab4d08d455fa9c5b14bf03a207e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/5d4c7ab4d08d455fa9c5b14bf03a207e 2024-11-19T15:14:08,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/e66ab7a7b75c473498dfc6d9522137b6 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/e66ab7a7b75c473498dfc6d9522137b6 2024-11-19T15:14:08,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/6a5b02a22a4e49f7b4ae7fdfbca4f249 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/6a5b02a22a4e49f7b4ae7fdfbca4f249 2024-11-19T15:14:08,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/60798c3f1ddc40d3bfec465e1334ba6b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/60798c3f1ddc40d3bfec465e1334ba6b 2024-11-19T15:14:08,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/30db31b174cc4f199ac4d1181241274a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/30db31b174cc4f199ac4d1181241274a 2024-11-19T15:14:08,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/eb628ca0f5e744e9abe685be08d214e1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/eb628ca0f5e744e9abe685be08d214e1 2024-11-19T15:14:08,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/935d72b396ef4e64a3770b31e4d22d8f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/935d72b396ef4e64a3770b31e4d22d8f 2024-11-19T15:14:08,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/d817fda905154e67863a8196521a13e4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/d817fda905154e67863a8196521a13e4 2024-11-19T15:14:08,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8010018295b24ac9bde6392f49bf2587 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8010018295b24ac9bde6392f49bf2587 2024-11-19T15:14:08,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/4536a3e533f0437e99ff7d7999df9c37 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/4536a3e533f0437e99ff7d7999df9c37 2024-11-19T15:14:08,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/94becc5056ee43dd84ddf486b0bc83fe to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/94becc5056ee43dd84ddf486b0bc83fe 2024-11-19T15:14:08,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8adc0e9942294697a8a46de6f00dbe0d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8adc0e9942294697a8a46de6f00dbe0d 2024-11-19T15:14:08,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/42d3b17fe00348b69eeff51008c21614 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/42d3b17fe00348b69eeff51008c21614 2024-11-19T15:14:08,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1deeed7441e34bc2a6f1d5d9430d4f9d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1deeed7441e34bc2a6f1d5d9430d4f9d 2024-11-19T15:14:08,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8ddebca81bd343b68e5b3aa85e97499a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/8ddebca81bd343b68e5b3aa85e97499a 2024-11-19T15:14:08,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/6ed564d41b244ef3a26c28cfcb514e52 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/6ed564d41b244ef3a26c28cfcb514e52 2024-11-19T15:14:08,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/718a627e671e47418b0cb2436dcf340f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/718a627e671e47418b0cb2436dcf340f 2024-11-19T15:14:08,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/259f72d143244a72af0520aeabd010a4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/259f72d143244a72af0520aeabd010a4 2024-11-19T15:14:08,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/bf53d35890684b7b825a89a3d8aca6a7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/bf53d35890684b7b825a89a3d8aca6a7 2024-11-19T15:14:08,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/b731c646230047e584be58259f453566 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/b731c646230047e584be58259f453566 2024-11-19T15:14:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T15:14:08,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/93e918556d50433daf707c499dc242d3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/93e918556d50433daf707c499dc242d3 2024-11-19T15:14:08,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/ea747ef0e6ce406c84b24bfc746b7339 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/ea747ef0e6ce406c84b24bfc746b7339 2024-11-19T15:14:08,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/073bbb945c164b7fb646752ac6c0ec2d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/073bbb945c164b7fb646752ac6c0ec2d 2024-11-19T15:14:08,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/da1349403ab74037bf68f911ddbf02d5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/da1349403ab74037bf68f911ddbf02d5 2024-11-19T15:14:08,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/fc1e922c003b423097c0fd4c43890be6 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/fc1e922c003b423097c0fd4c43890be6 2024-11-19T15:14:08,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/dc03c64df3e74698884ac5b3c1ce22d1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/dc03c64df3e74698884ac5b3c1ce22d1 2024-11-19T15:14:08,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/a5b7d2c055a14572a7e714afa6896f34 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/a5b7d2c055a14572a7e714afa6896f34 2024-11-19T15:14:08,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/7d0850d9fc094553a12294c1e9bdd31c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/7d0850d9fc094553a12294c1e9bdd31c 2024-11-19T15:14:08,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1dff0fa4c8554443b4d6206ff1ba098f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/1dff0fa4c8554443b4d6206ff1ba098f 2024-11-19T15:14:08,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/5ada67c2162f403db4fc0a55beb0c0b7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/5ada67c2162f403db4fc0a55beb0c0b7 2024-11-19T15:14:08,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/78a112a56064478286aa1952674b4e35, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/88fa628b233844c385103c8ee86cfbd2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/352a932d9c8641ada102ac4ce95b1eec, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/b7abc79f4f6444cc88887430949dc331, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/837720a0f21342bbabf42a0b6d34b31d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/78ab13b66687429294c6886b1f461300, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/6fced96c4dbb4b0b8d3ab92e719c134b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/d1b0d86c0b654392a06e3892c71ff7b9, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/ad650fea4c6c442aa2fe71fffc4912de, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/4714a78bb4a549d8b6729995450fb1c5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/8e95f6148e5f4b00ae4517b1cd0552ca, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/08db943566d340b9b97fb14f98c55473, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/57324c53aee549b4aa3be88536849757, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/65d6af4ef3f14b359485ae7a489ea3eb, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/770021b7a8934656b0754192c21f828b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/8d0f20c8ce734e5d9eb7d3833ff9e02f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/67b2f56a788d45be85d95a37c2d1d294, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/15fa3562efcc45a5b910dbb3e8f86c2b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/ba37445629a7479f8924a5b0b72c4c25, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/df547eb2507f4d2d87a398b0b3bd5026, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/3f7a64fa1afa4dc5bc33acdaacf18200, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/2c89aad715ad4771a23a26e199713eb8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/0df3e1f282a44bd08b510ecb40aad286, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/9199c7114cd94482927b7f5ac7574ccc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/0bafab0a94f848c28be02bf5cfb1e34a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/56ff621398b94bb686bda314881a3f78, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/1492e5b9ad86428f89aaa720f7d95d94, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/fb15900e07a94ee4ac7778cc2692ccff, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/da9b866449124e3ab3e4a75c79ae191b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/e11aaf22d52f4cb4a2497766bad5dad5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/b44866a3d4f5492293f19cbe1d2d1bbb, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/1bbeef8092a343e79cc30fa420f4443c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/6672dd9dbb2846b3b41071f7eb955442, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/47ef636fe42e4a76a16b136062f637e5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/f5c440fef5fa4b368b60b33b5f2ef070, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/94ae798e13e7493ea543fae21868a6c8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/436bb2a9f4cf4e179dac4381585af41f] to archive 2024-11-19T15:14:08,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:14:08,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/78a112a56064478286aa1952674b4e35 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/78a112a56064478286aa1952674b4e35 2024-11-19T15:14:08,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/88fa628b233844c385103c8ee86cfbd2 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/88fa628b233844c385103c8ee86cfbd2 2024-11-19T15:14:08,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/352a932d9c8641ada102ac4ce95b1eec to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/352a932d9c8641ada102ac4ce95b1eec 2024-11-19T15:14:08,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/b7abc79f4f6444cc88887430949dc331 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/b7abc79f4f6444cc88887430949dc331 2024-11-19T15:14:08,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/837720a0f21342bbabf42a0b6d34b31d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/837720a0f21342bbabf42a0b6d34b31d 2024-11-19T15:14:08,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/78ab13b66687429294c6886b1f461300 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/78ab13b66687429294c6886b1f461300 2024-11-19T15:14:08,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/6fced96c4dbb4b0b8d3ab92e719c134b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/6fced96c4dbb4b0b8d3ab92e719c134b 2024-11-19T15:14:08,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/d1b0d86c0b654392a06e3892c71ff7b9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/d1b0d86c0b654392a06e3892c71ff7b9 2024-11-19T15:14:08,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/ad650fea4c6c442aa2fe71fffc4912de to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/ad650fea4c6c442aa2fe71fffc4912de 2024-11-19T15:14:08,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/4714a78bb4a549d8b6729995450fb1c5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/4714a78bb4a549d8b6729995450fb1c5 2024-11-19T15:14:08,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/8e95f6148e5f4b00ae4517b1cd0552ca to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/8e95f6148e5f4b00ae4517b1cd0552ca 2024-11-19T15:14:08,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/08db943566d340b9b97fb14f98c55473 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/08db943566d340b9b97fb14f98c55473 2024-11-19T15:14:08,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/57324c53aee549b4aa3be88536849757 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/57324c53aee549b4aa3be88536849757 2024-11-19T15:14:08,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/65d6af4ef3f14b359485ae7a489ea3eb to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/65d6af4ef3f14b359485ae7a489ea3eb 2024-11-19T15:14:08,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/770021b7a8934656b0754192c21f828b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/770021b7a8934656b0754192c21f828b 2024-11-19T15:14:08,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/8d0f20c8ce734e5d9eb7d3833ff9e02f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/8d0f20c8ce734e5d9eb7d3833ff9e02f 2024-11-19T15:14:08,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/67b2f56a788d45be85d95a37c2d1d294 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/67b2f56a788d45be85d95a37c2d1d294 2024-11-19T15:14:08,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/15fa3562efcc45a5b910dbb3e8f86c2b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/15fa3562efcc45a5b910dbb3e8f86c2b 2024-11-19T15:14:08,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/ba37445629a7479f8924a5b0b72c4c25 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/ba37445629a7479f8924a5b0b72c4c25 2024-11-19T15:14:08,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/df547eb2507f4d2d87a398b0b3bd5026 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/df547eb2507f4d2d87a398b0b3bd5026 2024-11-19T15:14:08,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/3f7a64fa1afa4dc5bc33acdaacf18200 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/3f7a64fa1afa4dc5bc33acdaacf18200 2024-11-19T15:14:08,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/2c89aad715ad4771a23a26e199713eb8 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/2c89aad715ad4771a23a26e199713eb8 2024-11-19T15:14:08,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/0df3e1f282a44bd08b510ecb40aad286 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/0df3e1f282a44bd08b510ecb40aad286 2024-11-19T15:14:08,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/9199c7114cd94482927b7f5ac7574ccc to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/9199c7114cd94482927b7f5ac7574ccc 2024-11-19T15:14:08,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/0bafab0a94f848c28be02bf5cfb1e34a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/0bafab0a94f848c28be02bf5cfb1e34a 2024-11-19T15:14:08,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/56ff621398b94bb686bda314881a3f78 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/56ff621398b94bb686bda314881a3f78 2024-11-19T15:14:08,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/1492e5b9ad86428f89aaa720f7d95d94 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/1492e5b9ad86428f89aaa720f7d95d94 2024-11-19T15:14:08,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/fb15900e07a94ee4ac7778cc2692ccff to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/fb15900e07a94ee4ac7778cc2692ccff 2024-11-19T15:14:08,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/da9b866449124e3ab3e4a75c79ae191b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/da9b866449124e3ab3e4a75c79ae191b 2024-11-19T15:14:08,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/e11aaf22d52f4cb4a2497766bad5dad5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/e11aaf22d52f4cb4a2497766bad5dad5 2024-11-19T15:14:08,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/b44866a3d4f5492293f19cbe1d2d1bbb to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/b44866a3d4f5492293f19cbe1d2d1bbb 2024-11-19T15:14:08,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/1bbeef8092a343e79cc30fa420f4443c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/1bbeef8092a343e79cc30fa420f4443c 2024-11-19T15:14:08,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/6672dd9dbb2846b3b41071f7eb955442 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/6672dd9dbb2846b3b41071f7eb955442 2024-11-19T15:14:08,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/47ef636fe42e4a76a16b136062f637e5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/47ef636fe42e4a76a16b136062f637e5 2024-11-19T15:14:08,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/f5c440fef5fa4b368b60b33b5f2ef070 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/f5c440fef5fa4b368b60b33b5f2ef070 2024-11-19T15:14:08,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/94ae798e13e7493ea543fae21868a6c8 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/94ae798e13e7493ea543fae21868a6c8 2024-11-19T15:14:08,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/436bb2a9f4cf4e179dac4381585af41f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/436bb2a9f4cf4e179dac4381585af41f 2024-11-19T15:14:08,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/722ee79694394407b104c3be58b398fd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e1cdd24fbf874bdc95f3f0f1e475c57c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/c9e4924715f84e9f830a080ccfffd2c4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5f3078325fe84054a9b0de86ae535331, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2e09255af2af4c89b37329e3d762eda7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/dcb4426516504e94a7155b253a254574, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/613a1dc10133471aaa95f660395b369a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2adda2c1a4c64535a5c566d5da390a64, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/1744cf6d1ecd4f1abb44ec45f64822da, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/4145b77b37a24be6b6553931365b966f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/fc4e78e551ee4996a765f66de9263723, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/93c60c3182c8451686c4cee891cfebf6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/de15a198e73941e9a1d1eaf370a34e54, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5fc0379eba994831a6fbe808c2d62888, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/570af6493da04e2bb9d2dc526587cbd5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e8e10e97a0ba412d9cbc3a256a37a637, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/a395c1cc3ca6421d9f58efee8f10b3e0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/4e1a1eb3b9204862a62b55af00837717, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/d865538949e742438fa38c8114796f6f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/14dc77378b424e2ebaed0533becbfba0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9895c8360cc6465ea0d531d4d948342a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e73ddc039ee244d380f75f418bdcd49e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e2be859882f04f7a9520725f154e8680, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/67d85af8ccc746469a7b063c7a440c57, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/7181dcc3211d4383ab98f18e3cf6e297, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9d4222a955934a65bd3ba6300d085dc4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2cc81cbb94a6442aac0fcaac57c4b7ce, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/6c16990031da4f28a7da66c41ee11827, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/c17e7ddf8a7c4a07a9b4edb8c6168cb6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2784f57b9a01458baea904afa183c025, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/1ed68937360041088ab96c59c9b481dd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/97a5472d2d3e498d8e0d9156803d45e9, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/942119cf30da43a6920f61199e0dc1fc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/d59cca8734fb46abbddf2e05fad9cf3e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5260887ab0f24f35b721bbf1c1153423, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/7adba33f041e4217bc1605b0f381d13a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9671e72e093c4a3bb8d8e931634aaa8f] to archive 2024-11-19T15:14:08,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:14:08,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/722ee79694394407b104c3be58b398fd to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/722ee79694394407b104c3be58b398fd 2024-11-19T15:14:08,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e1cdd24fbf874bdc95f3f0f1e475c57c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e1cdd24fbf874bdc95f3f0f1e475c57c 2024-11-19T15:14:08,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/c9e4924715f84e9f830a080ccfffd2c4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/c9e4924715f84e9f830a080ccfffd2c4 2024-11-19T15:14:08,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5f3078325fe84054a9b0de86ae535331 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5f3078325fe84054a9b0de86ae535331 2024-11-19T15:14:08,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2e09255af2af4c89b37329e3d762eda7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2e09255af2af4c89b37329e3d762eda7 2024-11-19T15:14:08,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/dcb4426516504e94a7155b253a254574 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/dcb4426516504e94a7155b253a254574 2024-11-19T15:14:08,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/613a1dc10133471aaa95f660395b369a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/613a1dc10133471aaa95f660395b369a 2024-11-19T15:14:08,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2adda2c1a4c64535a5c566d5da390a64 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2adda2c1a4c64535a5c566d5da390a64 2024-11-19T15:14:08,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/1744cf6d1ecd4f1abb44ec45f64822da to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/1744cf6d1ecd4f1abb44ec45f64822da 2024-11-19T15:14:08,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/4145b77b37a24be6b6553931365b966f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/4145b77b37a24be6b6553931365b966f 2024-11-19T15:14:08,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/fc4e78e551ee4996a765f66de9263723 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/fc4e78e551ee4996a765f66de9263723 2024-11-19T15:14:08,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/93c60c3182c8451686c4cee891cfebf6 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/93c60c3182c8451686c4cee891cfebf6 2024-11-19T15:14:08,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/de15a198e73941e9a1d1eaf370a34e54 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/de15a198e73941e9a1d1eaf370a34e54 2024-11-19T15:14:08,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5fc0379eba994831a6fbe808c2d62888 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5fc0379eba994831a6fbe808c2d62888 2024-11-19T15:14:08,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/570af6493da04e2bb9d2dc526587cbd5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/570af6493da04e2bb9d2dc526587cbd5 2024-11-19T15:14:08,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e8e10e97a0ba412d9cbc3a256a37a637 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e8e10e97a0ba412d9cbc3a256a37a637 2024-11-19T15:14:08,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/a395c1cc3ca6421d9f58efee8f10b3e0 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/a395c1cc3ca6421d9f58efee8f10b3e0 2024-11-19T15:14:08,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/4e1a1eb3b9204862a62b55af00837717 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/4e1a1eb3b9204862a62b55af00837717 2024-11-19T15:14:08,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/d865538949e742438fa38c8114796f6f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/d865538949e742438fa38c8114796f6f 2024-11-19T15:14:08,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/14dc77378b424e2ebaed0533becbfba0 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/14dc77378b424e2ebaed0533becbfba0 2024-11-19T15:14:08,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9895c8360cc6465ea0d531d4d948342a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9895c8360cc6465ea0d531d4d948342a 2024-11-19T15:14:08,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e73ddc039ee244d380f75f418bdcd49e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e73ddc039ee244d380f75f418bdcd49e 2024-11-19T15:14:08,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e2be859882f04f7a9520725f154e8680 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e2be859882f04f7a9520725f154e8680 2024-11-19T15:14:08,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/67d85af8ccc746469a7b063c7a440c57 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/67d85af8ccc746469a7b063c7a440c57 2024-11-19T15:14:08,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/7181dcc3211d4383ab98f18e3cf6e297 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/7181dcc3211d4383ab98f18e3cf6e297 2024-11-19T15:14:08,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9d4222a955934a65bd3ba6300d085dc4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9d4222a955934a65bd3ba6300d085dc4 2024-11-19T15:14:08,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2cc81cbb94a6442aac0fcaac57c4b7ce to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2cc81cbb94a6442aac0fcaac57c4b7ce 2024-11-19T15:14:08,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/6c16990031da4f28a7da66c41ee11827 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/6c16990031da4f28a7da66c41ee11827 2024-11-19T15:14:08,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/c17e7ddf8a7c4a07a9b4edb8c6168cb6 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/c17e7ddf8a7c4a07a9b4edb8c6168cb6 2024-11-19T15:14:08,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2784f57b9a01458baea904afa183c025 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/2784f57b9a01458baea904afa183c025 2024-11-19T15:14:08,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/1ed68937360041088ab96c59c9b481dd to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/1ed68937360041088ab96c59c9b481dd 2024-11-19T15:14:08,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/97a5472d2d3e498d8e0d9156803d45e9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/97a5472d2d3e498d8e0d9156803d45e9 2024-11-19T15:14:08,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/942119cf30da43a6920f61199e0dc1fc to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/942119cf30da43a6920f61199e0dc1fc 2024-11-19T15:14:08,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/d59cca8734fb46abbddf2e05fad9cf3e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/d59cca8734fb46abbddf2e05fad9cf3e 2024-11-19T15:14:08,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5260887ab0f24f35b721bbf1c1153423 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/5260887ab0f24f35b721bbf1c1153423 2024-11-19T15:14:08,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/7adba33f041e4217bc1605b0f381d13a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/7adba33f041e4217bc1605b0f381d13a 2024-11-19T15:14:08,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9671e72e093c4a3bb8d8e931634aaa8f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/9671e72e093c4a3bb8d8e931634aaa8f 2024-11-19T15:14:08,828 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/recovered.edits/800.seqid, newMaxSeqId=800, maxSeqId=1 2024-11-19T15:14:08,829 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. 2024-11-19T15:14:08,830 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 9a10ee694987b867f252a8f623c69480: Waiting for close lock at 1732029248532Running coprocessor pre-close hooks at 1732029248532Disabling compacts and flushes for region at 1732029248532Disabling writes for close at 1732029248533 (+1 ms)Obtaining lock to block concurrent updates at 1732029248533Preparing flush snapshotting stores in 9a10ee694987b867f252a8f623c69480 at 1732029248533Finished memstore snapshotting TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732029248534 (+1 ms)Flushing stores of TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480. at 1732029248534Flushing 9a10ee694987b867f252a8f623c69480/A: creating writer at 1732029248534Flushing 9a10ee694987b867f252a8f623c69480/A: appending metadata at 1732029248539 (+5 ms)Flushing 9a10ee694987b867f252a8f623c69480/A: closing flushed file at 1732029248539Flushing 9a10ee694987b867f252a8f623c69480/B: creating writer at 1732029248559 (+20 ms)Flushing 9a10ee694987b867f252a8f623c69480/B: appending metadata at 1732029248563 (+4 ms)Flushing 9a10ee694987b867f252a8f623c69480/B: closing flushed file at 1732029248563Flushing 9a10ee694987b867f252a8f623c69480/C: creating writer at 1732029248578 (+15 ms)Flushing 9a10ee694987b867f252a8f623c69480/C: appending metadata at 1732029248582 (+4 ms)Flushing 9a10ee694987b867f252a8f623c69480/C: closing flushed file at 1732029248582Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e7d083: reopening flushed file at 1732029248595 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cc631ba: reopening flushed file at 1732029248602 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c9ba5b: reopening flushed file at 1732029248608 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 9a10ee694987b867f252a8f623c69480 in 86ms, sequenceid=797, compaction requested=false at 1732029248619 (+11 ms)Writing region close event to WAL at 1732029248825 (+206 ms)Running coprocessor post-close hooks at 1732029248829 (+4 ms)Closed at 1732029248829 2024-11-19T15:14:08,833 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 9a10ee694987b867f252a8f623c69480 2024-11-19T15:14:08,834 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9a10ee694987b867f252a8f623c69480, regionState=CLOSED 2024-11-19T15:14:08,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:14:08,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-19T15:14:08,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 9a10ee694987b867f252a8f623c69480, server=50570b5b74e7,46557,1732029218639 in 464 msec 2024-11-19T15:14:08,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-19T15:14:08,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a10ee694987b867f252a8f623c69480, UNASSIGN in 474 msec 2024-11-19T15:14:08,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-19T15:14:08,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 483 msec 2024-11-19T15:14:08,844 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029248844"}]},"ts":"1732029248844"} 2024-11-19T15:14:08,846 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T15:14:08,846 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T15:14:08,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 501 msec 2024-11-19T15:14:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T15:14:08,984 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:08,987 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T15:14:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:08,998 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:09,000 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T15:14:09,016 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480 2024-11-19T15:14:09,022 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/recovered.edits] 2024-11-19T15:14:09,025 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/04f55619a7a74c2aa8544e9637009dd7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/04f55619a7a74c2aa8544e9637009dd7 2024-11-19T15:14:09,027 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/cb854e4161004183ab47ffb7a5f6af9c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/A/cb854e4161004183ab47ffb7a5f6af9c 2024-11-19T15:14:09,030 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/5f2334d69ebd45b7848ef2fbe74bcfa7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/5f2334d69ebd45b7848ef2fbe74bcfa7 2024-11-19T15:14:09,031 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/de51146be00f44abb6ff5f60e0a14edf to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/B/de51146be00f44abb6ff5f60e0a14edf 2024-11-19T15:14:09,034 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/10c23fd7b5ed4ba282f64332b0200d68 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/10c23fd7b5ed4ba282f64332b0200d68 2024-11-19T15:14:09,035 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e49af6e75f2641e0b722dfaa7a705d12 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/C/e49af6e75f2641e0b722dfaa7a705d12 2024-11-19T15:14:09,038 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/recovered.edits/800.seqid to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480/recovered.edits/800.seqid 2024-11-19T15:14:09,039 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9a10ee694987b867f252a8f623c69480 2024-11-19T15:14:09,039 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T15:14:09,045 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:09,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-19T15:14:09,058 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T15:14:09,110 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T15:14:09,112 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:09,112 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T15:14:09,112 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732029249112"}]},"ts":"9223372036854775807"} 2024-11-19T15:14:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T15:14:09,128 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T15:14:09,128 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9a10ee694987b867f252a8f623c69480, NAME => 'TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T15:14:09,128 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T15:14:09,129 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732029249128"}]},"ts":"9223372036854775807"} 2024-11-19T15:14:09,134 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T15:14:09,136 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:09,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 148 msec 2024-11-19T15:14:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T15:14:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T15:14:09,325 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:09,338 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1860180247_22 at /127.0.0.1:49252 [Waiting for operation #261] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;50570b5b74e7:46557-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1860180247_22 at /127.0.0.1:46404 [Waiting for operation #228] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1594712618_22 at /127.0.0.1:46306 [Waiting for operation #245] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=360 (was 269) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8578 (was 8400) - AvailableMemoryMB LEAK? - 2024-11-19T15:14:09,349 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=360, ProcessCount=11, AvailableMemoryMB=8577 2024-11-19T15:14:09,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T15:14:09,351 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T15:14:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:09,354 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T15:14:09,354 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:09,354 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-19T15:14:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T15:14:09,355 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T15:14:09,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741954_1130 (size=1061) 2024-11-19T15:14:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T15:14:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T15:14:09,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8 2024-11-19T15:14:09,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741955_1131 (size=53) 2024-11-19T15:14:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T15:14:10,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:14:10,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f012bfcce8bd6aef12a49ca6ef302f96, disabling compactions & flushes 2024-11-19T15:14:10,175 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:10,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:10,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. after waiting 0 ms 2024-11-19T15:14:10,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:10,175 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:10,175 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f012bfcce8bd6aef12a49ca6ef302f96: Waiting for close lock at 1732029250175Disabling compacts and flushes for region at 1732029250175Disabling writes for close at 1732029250175Writing region close event to WAL at 1732029250175Closed at 1732029250175 2024-11-19T15:14:10,176 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T15:14:10,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732029250176"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732029250176"}]},"ts":"1732029250176"} 2024-11-19T15:14:10,178 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T15:14:10,179 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T15:14:10,179 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029250179"}]},"ts":"1732029250179"} 2024-11-19T15:14:10,180 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T15:14:10,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, ASSIGN}] 2024-11-19T15:14:10,182 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, ASSIGN 2024-11-19T15:14:10,183 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, ASSIGN; state=OFFLINE, location=50570b5b74e7,46557,1732029218639; forceNewPlan=false, retain=false 2024-11-19T15:14:10,333 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f012bfcce8bd6aef12a49ca6ef302f96, regionState=OPENING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:10,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, ASSIGN because future has completed 2024-11-19T15:14:10,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:14:10,492 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:10,492 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} 2024-11-19T15:14:10,493 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,493 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:14:10,493 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,493 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T15:14:10,495 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,496 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:14:10,497 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f012bfcce8bd6aef12a49ca6ef302f96 columnFamilyName A 2024-11-19T15:14:10,497 DEBUG [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:10,497 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.HStore(327): Store=f012bfcce8bd6aef12a49ca6ef302f96/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:14:10,497 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,498 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:14:10,499 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f012bfcce8bd6aef12a49ca6ef302f96 columnFamilyName B 2024-11-19T15:14:10,499 DEBUG [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:10,499 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.HStore(327): Store=f012bfcce8bd6aef12a49ca6ef302f96/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:14:10,499 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,500 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:14:10,500 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f012bfcce8bd6aef12a49ca6ef302f96 columnFamilyName C 2024-11-19T15:14:10,500 DEBUG [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:10,501 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.HStore(327): Store=f012bfcce8bd6aef12a49ca6ef302f96/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:14:10,501 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:10,501 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,501 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,502 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,502 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,503 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,503 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T15:14:10,504 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,506 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T15:14:10,506 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened f012bfcce8bd6aef12a49ca6ef302f96; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72445155, jitterRate=0.0795169323682785}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T15:14:10,506 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:10,508 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for f012bfcce8bd6aef12a49ca6ef302f96: Running coprocessor pre-open hook at 1732029250493Writing region info on filesystem at 1732029250493Initializing all the Stores at 1732029250494 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029250494Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029250495 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029250495Cleaning up temporary data from old regions at 1732029250503 (+8 ms)Running coprocessor post-open hooks at 1732029250506 (+3 ms)Region opened successfully at 1732029250508 (+2 ms) 2024-11-19T15:14:10,509 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., pid=34, masterSystemTime=1732029250489 2024-11-19T15:14:10,511 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:10,511 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:10,511 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f012bfcce8bd6aef12a49ca6ef302f96, regionState=OPEN, openSeqNum=2, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:10,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:14:10,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-19T15:14:10,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 in 177 msec 2024-11-19T15:14:10,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-19T15:14:10,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, ASSIGN in 336 msec 2024-11-19T15:14:10,520 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T15:14:10,520 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029250520"}]},"ts":"1732029250520"} 2024-11-19T15:14:10,522 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T15:14:10,522 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T15:14:10,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-11-19T15:14:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T15:14:11,504 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:11,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:11,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:11,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:11,507 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eaa07b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:11,509 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7e2b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:11,510 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:11,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:11,514 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T15:14:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 50570b5b74e7,39501,1732029217890 2024-11-19T15:14:11,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@557415e3 2024-11-19T15:14:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T15:14:11,515 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T15:14:11,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T15:14:11,520 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T15:14:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:11,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741956_1132 (size=1097) 2024-11-19T15:14:11,940 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T15:14:11,940 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T15:14:11,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T15:14:11,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, REOPEN/MOVE}] 2024-11-19T15:14:11,951 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, REOPEN/MOVE 2024-11-19T15:14:11,952 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=f012bfcce8bd6aef12a49ca6ef302f96, regionState=CLOSING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:11,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, REOPEN/MOVE because future has completed 2024-11-19T15:14:11,954 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T15:14:11,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:14:12,106 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,106 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T15:14:12,107 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing f012bfcce8bd6aef12a49ca6ef302f96, disabling compactions & flushes 2024-11-19T15:14:12,107 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,107 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,107 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. after waiting 0 ms 2024-11-19T15:14:12,107 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,110 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T15:14:12,111 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,111 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for f012bfcce8bd6aef12a49ca6ef302f96: Waiting for close lock at 1732029252106Running coprocessor pre-close hooks at 1732029252106Disabling compacts and flushes for region at 1732029252106Disabling writes for close at 1732029252107 (+1 ms)Writing region close event to WAL at 1732029252107Running coprocessor post-close hooks at 1732029252111 (+4 ms)Closed at 1732029252111 2024-11-19T15:14:12,111 WARN [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: f012bfcce8bd6aef12a49ca6ef302f96 to self. 2024-11-19T15:14:12,113 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,114 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=f012bfcce8bd6aef12a49ca6ef302f96, regionState=CLOSED 2024-11-19T15:14:12,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:14:12,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-19T15:14:12,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 in 162 msec 2024-11-19T15:14:12,118 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, REOPEN/MOVE; state=CLOSED, location=50570b5b74e7,46557,1732029218639; forceNewPlan=false, retain=true 2024-11-19T15:14:12,269 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=f012bfcce8bd6aef12a49ca6ef302f96, regionState=OPENING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, REOPEN/MOVE because future has completed 2024-11-19T15:14:12,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:14:12,427 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,427 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} 2024-11-19T15:14:12,427 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,427 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:14:12,427 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,427 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,429 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,430 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:14:12,435 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f012bfcce8bd6aef12a49ca6ef302f96 columnFamilyName A 2024-11-19T15:14:12,437 DEBUG [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:12,437 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.HStore(327): Store=f012bfcce8bd6aef12a49ca6ef302f96/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:14:12,438 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,438 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:14:12,438 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f012bfcce8bd6aef12a49ca6ef302f96 columnFamilyName B 2024-11-19T15:14:12,439 DEBUG [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:12,439 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.HStore(327): Store=f012bfcce8bd6aef12a49ca6ef302f96/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:14:12,439 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,439 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:14:12,440 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f012bfcce8bd6aef12a49ca6ef302f96 columnFamilyName C 2024-11-19T15:14:12,440 DEBUG [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:12,440 INFO [StoreOpener-f012bfcce8bd6aef12a49ca6ef302f96-1 {}] regionserver.HStore(327): Store=f012bfcce8bd6aef12a49ca6ef302f96/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:14:12,440 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,440 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,441 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,442 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,443 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,443 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,443 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T15:14:12,445 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,445 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened f012bfcce8bd6aef12a49ca6ef302f96; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69561393, jitterRate=0.036545529961586}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T15:14:12,446 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,447 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for f012bfcce8bd6aef12a49ca6ef302f96: Running coprocessor pre-open hook at 1732029252427Writing region info on filesystem at 1732029252427Initializing all the Stores at 1732029252428 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029252428Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029252428Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029252428Cleaning up temporary data from old regions at 1732029252443 (+15 ms)Running coprocessor post-open hooks at 1732029252446 (+3 ms)Region opened successfully at 1732029252447 (+1 ms) 2024-11-19T15:14:12,448 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., pid=39, masterSystemTime=1732029252423 2024-11-19T15:14:12,450 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,450 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,450 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=f012bfcce8bd6aef12a49ca6ef302f96, regionState=OPEN, openSeqNum=5, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:14:12,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-19T15:14:12,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 in 181 msec 2024-11-19T15:14:12,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-19T15:14:12,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, REOPEN/MOVE in 504 msec 2024-11-19T15:14:12,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-19T15:14:12,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 514 msec 2024-11-19T15:14:12,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 937 msec 2024-11-19T15:14:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-19T15:14:12,463 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:12,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:12,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:12,465 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b03771b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:12,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:12,467 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39fc79b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:12,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:12,469 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6df162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:12,470 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d70833f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,472 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:12,472 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:12,472 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@468c89a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:12,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:12,474 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f328861, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,475 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:12,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:12,476 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89db025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:12,477 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:12,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6f42a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,478 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:12,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:12,479 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:12,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:12,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd974f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:12,481 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707735aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:12,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:12,483 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384fa90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:12,484 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68029560, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:12,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:12,486 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@778405a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:12,487 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:12,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af31bce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:12,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:12,488 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:12,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:12,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:12,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66719043, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:12,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:12,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:12,490 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:12,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72737, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:12,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:12,491 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28c24289, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:12,493 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6096a572, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:12,494 DEBUG [Thread-674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:12,494 DEBUG [Thread-665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:12,494 DEBUG [Thread-663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:12,494 DEBUG [Thread-670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:12,495 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:12,495 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:12,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:12,495 DEBUG [Thread-668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:12,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:12,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:12,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:12,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:12,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:12,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-19T15:14:12,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:12,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:12,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:12,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:12,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:12,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:12,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:12,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:12,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T15:14:12,498 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:12,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:12,499 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:12,499 DEBUG [Thread-657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:12,499 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:12,499 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:12,499 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:12,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:14:12,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:14:12,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:12,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:14:12,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:14:12,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:12,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:14:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:14:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:14:12,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:14:12,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:14:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T15:14:12,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:12,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:12,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:12,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:12,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:12,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:12,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197f88bfa2360b418e8d3570d32583aba6_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029252519/Put/seqid=0 2024-11-19T15:14:12,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741957_1133 (size=12154) 2024-11-19T15:14:12,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029312576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029312578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1732029252557, max timestamp=1732029252583Num uniques 859; 2024-11-19T15:14:12,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-19T15:14:12,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029312581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1732029252557, max timestamp=1732029252583Num uniques 859; 2024-11-19T15:14:12,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1732029252557, max timestamp=1732029252583Num uniques 859; 2024-11-19T15:14:12,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-19T15:14:12,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-19T15:14:12,591 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,597 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197f88bfa2360b418e8d3570d32583aba6_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197f88bfa2360b418e8d3570d32583aba6_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:12,600 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/d3db53e766ba49af813d962c6f990365, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T15:14:12,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/d3db53e766ba49af813d962c6f990365 is 175, key is test_row_0/A:col10/1732029252519/Put/seqid=0 2024-11-19T15:14:12,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741958_1134 (size=30955) 2024-11-19T15:14:12,627 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/d3db53e766ba49af813d962c6f990365 2024-11-19T15:14:12,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:12,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:12,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:12,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:12,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029312661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029312661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/fe968d1ad042478e88e9444071b1672a is 50, key is test_row_0/B:col10/1732029252519/Put/seqid=0 2024-11-19T15:14:12,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741959_1135 (size=12001) 2024-11-19T15:14:12,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/fe968d1ad042478e88e9444071b1672a 2024-11-19T15:14:12,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029312695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029312695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029312695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/c6247f6e18124c71a2e2a83ef2bb25a9 is 50, key is test_row_0/C:col10/1732029252519/Put/seqid=0 2024-11-19T15:14:12,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741960_1136 (size=12001) 2024-11-19T15:14:12,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029312784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029312784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:12,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:12,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:12,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T15:14:12,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029312905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029312905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029312905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:12,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:12,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:12,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:12,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:12,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029312995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029312995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:12,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:13,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:13,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:13,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:13,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T15:14:13,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/c6247f6e18124c71a2e2a83ef2bb25a9 2024-11-19T15:14:13,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/d3db53e766ba49af813d962c6f990365 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/d3db53e766ba49af813d962c6f990365 2024-11-19T15:14:13,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/d3db53e766ba49af813d962c6f990365, entries=150, sequenceid=15, filesize=30.2 K 2024-11-19T15:14:13,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/fe968d1ad042478e88e9444071b1672a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/fe968d1ad042478e88e9444071b1672a 2024-11-19T15:14:13,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/fe968d1ad042478e88e9444071b1672a, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T15:14:13,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/c6247f6e18124c71a2e2a83ef2bb25a9 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/c6247f6e18124c71a2e2a83ef2bb25a9 2024-11-19T15:14:13,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/c6247f6e18124c71a2e2a83ef2bb25a9, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T15:14:13,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for f012bfcce8bd6aef12a49ca6ef302f96 in 636ms, sequenceid=15, compaction requested=false 2024-11-19T15:14:13,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:13,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=147.60 KB heapSize=396.70 KB 2024-11-19T15:14:13,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:13,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:13,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:13,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:13,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:13,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:13,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196c72976759de45718250c9c63c81e08a_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029252656/Put/seqid=0 2024-11-19T15:14:13,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029313237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029313249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029313250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741961_1137 (size=17034) 2024-11-19T15:14:13,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:13,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029313304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029313304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029313354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029313364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029313364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:13,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:13,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:13,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:13,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029313564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029313575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029313575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:13,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:13,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:13,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:13,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T15:14:13,671 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:13,678 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196c72976759de45718250c9c63c81e08a_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196c72976759de45718250c9c63c81e08a_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:13,680 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/c1af040d39484063991a77015d20dcb2, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:13,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/c1af040d39484063991a77015d20dcb2 is 175, key is test_row_0/A:col10/1732029252656/Put/seqid=0 2024-11-19T15:14:13,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741962_1138 (size=48139) 2024-11-19T15:14:13,700 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/c1af040d39484063991a77015d20dcb2 2024-11-19T15:14:13,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/775b1a42bce04c0bb3decc50f46ff359 is 50, key is test_row_0/B:col10/1732029252656/Put/seqid=0 2024-11-19T15:14:13,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741963_1139 (size=12001) 2024-11-19T15:14:13,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/775b1a42bce04c0bb3decc50f46ff359 2024-11-19T15:14:13,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:13,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:13,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:13,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:13,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/4e86e10495e7459a88f68a346502c5db is 50, key is test_row_0/C:col10/1732029252656/Put/seqid=0 2024-11-19T15:14:13,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741964_1140 (size=12001) 2024-11-19T15:14:13,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/4e86e10495e7459a88f68a346502c5db 2024-11-19T15:14:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/c1af040d39484063991a77015d20dcb2 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c1af040d39484063991a77015d20dcb2 2024-11-19T15:14:13,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c1af040d39484063991a77015d20dcb2, entries=250, sequenceid=49, filesize=47.0 K 2024-11-19T15:14:13,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/775b1a42bce04c0bb3decc50f46ff359 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/775b1a42bce04c0bb3decc50f46ff359 2024-11-19T15:14:13,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/775b1a42bce04c0bb3decc50f46ff359, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T15:14:13,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/4e86e10495e7459a88f68a346502c5db as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/4e86e10495e7459a88f68a346502c5db 2024-11-19T15:14:13,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/4e86e10495e7459a88f68a346502c5db, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T15:14:13,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~416.58 KB/426576, currentSize=40.25 KB/41220 for f012bfcce8bd6aef12a49ca6ef302f96 in 567ms, sequenceid=49, compaction requested=false 2024-11-19T15:14:13,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:13,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:13,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:14:13,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:13,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:13,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:13,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:13,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:13,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:13,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a739134e44f14ae1a5a9164258c8b297_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029253241/Put/seqid=0 2024-11-19T15:14:13,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741965_1141 (size=14594) 2024-11-19T15:14:13,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:13,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:13,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:13,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:13,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029313884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029313884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:13,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732029253846, max timestamp=1732029253885Num uniques 859; 2024-11-19T15:14:13,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029313884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029253846, max timestamp=1732029253885Num uniques 859; 2024-11-19T15:14:13,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029253846, max timestamp=1732029253885Num uniques 859; 2024-11-19T15:14:13,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:13,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029313945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029313947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:13,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029313994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:14,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:14,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:14,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029314054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:14,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029314056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:14,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:14,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:14,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029314204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:14,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029314264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:14,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029314264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:14,279 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:14,284 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a739134e44f14ae1a5a9164258c8b297_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a739134e44f14ae1a5a9164258c8b297_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:14,285 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/24a698f38c2a4c0dbe035eed759dfdc0, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:14,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/24a698f38c2a4c0dbe035eed759dfdc0 is 175, key is test_row_0/A:col10/1732029253241/Put/seqid=0 2024-11-19T15:14:14,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741966_1142 (size=39549) 2024-11-19T15:14:14,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:14,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:14,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,376 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T15:14:14,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029314405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:14,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:14,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029314405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:14,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:14,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:14,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029314514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:14,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029314574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:14,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029314575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:14,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T15:14:14,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:14,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:14,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:14,691 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/24a698f38c2a4c0dbe035eed759dfdc0 2024-11-19T15:14:14,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/fb770ebf0ae0429f918b58e1cdfdd8e7 is 50, key is test_row_0/B:col10/1732029253241/Put/seqid=0 2024-11-19T15:14:14,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741967_1143 (size=12001) 2024-11-19T15:14:14,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/fb770ebf0ae0429f918b58e1cdfdd8e7 2024-11-19T15:14:14,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/7f7c93095580419f842189d2405d0394 is 50, key is test_row_0/C:col10/1732029253241/Put/seqid=0 2024-11-19T15:14:14,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741968_1144 (size=12001) 2024-11-19T15:14:14,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/7f7c93095580419f842189d2405d0394 2024-11-19T15:14:14,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/24a698f38c2a4c0dbe035eed759dfdc0 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/24a698f38c2a4c0dbe035eed759dfdc0 2024-11-19T15:14:14,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/24a698f38c2a4c0dbe035eed759dfdc0, entries=200, sequenceid=60, filesize=38.6 K 2024-11-19T15:14:14,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/fb770ebf0ae0429f918b58e1cdfdd8e7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/fb770ebf0ae0429f918b58e1cdfdd8e7 2024-11-19T15:14:14,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/fb770ebf0ae0429f918b58e1cdfdd8e7, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T15:14:14,763 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T15:14:14,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/7f7c93095580419f842189d2405d0394 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/7f7c93095580419f842189d2405d0394 2024-11-19T15:14:14,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/7f7c93095580419f842189d2405d0394, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T15:14:14,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f012bfcce8bd6aef12a49ca6ef302f96 in 954ms, sequenceid=60, compaction requested=true 2024-11-19T15:14:14,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:14,772 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:14,772 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:14,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:14,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:14,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:14,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:14,775 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:14,775 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/B is initiating minor compaction (all files) 2024-11-19T15:14:14,776 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/B in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,776 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:14,776 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/fe968d1ad042478e88e9444071b1672a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/775b1a42bce04c0bb3decc50f46ff359, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/fb770ebf0ae0429f918b58e1cdfdd8e7] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=35.2 K 2024-11-19T15:14:14,776 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/A is initiating minor compaction (all files) 2024-11-19T15:14:14,776 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/A in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,776 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/d3db53e766ba49af813d962c6f990365, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c1af040d39484063991a77015d20dcb2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/24a698f38c2a4c0dbe035eed759dfdc0] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=115.9 K 2024-11-19T15:14:14,776 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting fe968d1ad042478e88e9444071b1672a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732029252504 2024-11-19T15:14:14,776 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,776 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/d3db53e766ba49af813d962c6f990365, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c1af040d39484063991a77015d20dcb2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/24a698f38c2a4c0dbe035eed759dfdc0] 2024-11-19T15:14:14,777 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 775b1a42bce04c0bb3decc50f46ff359, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732029252656 2024-11-19T15:14:14,777 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3db53e766ba49af813d962c6f990365, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732029252504 2024-11-19T15:14:14,778 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1af040d39484063991a77015d20dcb2, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732029252645 2024-11-19T15:14:14,778 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting fb770ebf0ae0429f918b58e1cdfdd8e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029253241 2024-11-19T15:14:14,779 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24a698f38c2a4c0dbe035eed759dfdc0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029253233 2024-11-19T15:14:14,791 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:14,792 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/3c5b318be2364ded9d4335d30d15fde4 is 50, key is test_row_0/B:col10/1732029253241/Put/seqid=0 2024-11-19T15:14:14,798 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:14,805 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119792f7fe8303a4c53a12e809fe975421c_f012bfcce8bd6aef12a49ca6ef302f96 store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:14,810 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119792f7fe8303a4c53a12e809fe975421c_f012bfcce8bd6aef12a49ca6ef302f96, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:14,812 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119792f7fe8303a4c53a12e809fe975421c_f012bfcce8bd6aef12a49ca6ef302f96 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:14,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T15:14:14,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,814 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T15:14:14,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:14,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:14,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:14,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:14,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:14,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:14,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741969_1145 (size=12104) 2024-11-19T15:14:14,829 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/3c5b318be2364ded9d4335d30d15fde4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/3c5b318be2364ded9d4335d30d15fde4 2024-11-19T15:14:14,845 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/B of f012bfcce8bd6aef12a49ca6ef302f96 into 3c5b318be2364ded9d4335d30d15fde4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:14,845 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:14,845 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/B, priority=13, startTime=1732029254772; duration=0sec 2024-11-19T15:14:14,845 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:14,845 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:B 2024-11-19T15:14:14,845 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:14,847 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:14,847 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/C is initiating minor compaction (all files) 2024-11-19T15:14:14,847 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/C in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:14,847 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/c6247f6e18124c71a2e2a83ef2bb25a9, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/4e86e10495e7459a88f68a346502c5db, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/7f7c93095580419f842189d2405d0394] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=35.2 K 2024-11-19T15:14:14,847 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting c6247f6e18124c71a2e2a83ef2bb25a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732029252504 2024-11-19T15:14:14,848 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e86e10495e7459a88f68a346502c5db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732029252656 2024-11-19T15:14:14,851 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f7c93095580419f842189d2405d0394, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029253241 2024-11-19T15:14:14,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741970_1146 (size=4469) 2024-11-19T15:14:14,867 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#A#compaction#127 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:14,868 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/c7569166fd5a4673bedbe6cee0e170a1 is 175, key is test_row_0/A:col10/1732029253241/Put/seqid=0 2024-11-19T15:14:14,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119be85d1ab30984896806fcab91d06a707_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029253931/Put/seqid=0 2024-11-19T15:14:14,880 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#C#compaction#129 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:14,880 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/6da625c8ae184472859d96d9de39d321 is 50, key is test_row_0/C:col10/1732029253241/Put/seqid=0 2024-11-19T15:14:14,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741971_1147 (size=31058) 2024-11-19T15:14:14,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741972_1148 (size=12154) 2024-11-19T15:14:14,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741973_1149 (size=12104) 2024-11-19T15:14:14,921 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/6da625c8ae184472859d96d9de39d321 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/6da625c8ae184472859d96d9de39d321 2024-11-19T15:14:14,929 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/C of f012bfcce8bd6aef12a49ca6ef302f96 into 6da625c8ae184472859d96d9de39d321(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:14,929 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:14,929 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/C, priority=13, startTime=1732029254773; duration=0sec 2024-11-19T15:14:14,929 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:14,929 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:C 2024-11-19T15:14:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:15,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:15,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029315070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:15,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:15,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029315094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029315094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:15,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029315184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:15,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:15,305 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119be85d1ab30984896806fcab91d06a707_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119be85d1ab30984896806fcab91d06a707_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:15,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/2662366243764452b21379ffdfedf2b5, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:15,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/2662366243764452b21379ffdfedf2b5 is 175, key is test_row_0/A:col10/1732029253931/Put/seqid=0 2024-11-19T15:14:15,311 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/c7569166fd5a4673bedbe6cee0e170a1 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c7569166fd5a4673bedbe6cee0e170a1 2024-11-19T15:14:15,321 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/A of f012bfcce8bd6aef12a49ca6ef302f96 into c7569166fd5a4673bedbe6cee0e170a1(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:15,321 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:15,321 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/A, priority=13, startTime=1732029254771; duration=0sec 2024-11-19T15:14:15,321 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:15,321 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:A 2024-11-19T15:14:15,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741974_1150 (size=30955) 2024-11-19T15:14:15,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029315394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:15,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029315424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:15,427 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in f012bfcce8bd6aef12a49ca6ef302f96 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:15,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029315434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:15,436 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in f012bfcce8bd6aef12a49ca6ef302f96 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:15,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029315705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:15,724 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/2662366243764452b21379ffdfedf2b5 2024-11-19T15:14:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/d88fc8e0d89948c2802f66fd46883d9c is 50, key is test_row_0/B:col10/1732029253931/Put/seqid=0 2024-11-19T15:14:15,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741975_1151 (size=12001) 2024-11-19T15:14:16,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029316104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:16,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029316105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:16,108 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in f012bfcce8bd6aef12a49ca6ef302f96 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:16,109 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in f012bfcce8bd6aef12a49ca6ef302f96 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:16,139 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/d88fc8e0d89948c2802f66fd46883d9c 2024-11-19T15:14:16,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/f84bc52d160b4e7fa9c57a3c7efe26d7 is 50, key is test_row_0/C:col10/1732029253931/Put/seqid=0 2024-11-19T15:14:16,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741976_1152 (size=12001) 2024-11-19T15:14:16,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029316224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:16,554 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/f84bc52d160b4e7fa9c57a3c7efe26d7 2024-11-19T15:14:16,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/2662366243764452b21379ffdfedf2b5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2662366243764452b21379ffdfedf2b5 2024-11-19T15:14:16,567 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2662366243764452b21379ffdfedf2b5, entries=150, sequenceid=95, filesize=30.2 K 2024-11-19T15:14:16,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/d88fc8e0d89948c2802f66fd46883d9c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/d88fc8e0d89948c2802f66fd46883d9c 2024-11-19T15:14:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,574 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/d88fc8e0d89948c2802f66fd46883d9c, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T15:14:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/f84bc52d160b4e7fa9c57a3c7efe26d7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/f84bc52d160b4e7fa9c57a3c7efe26d7 2024-11-19T15:14:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,581 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/f84bc52d160b4e7fa9c57a3c7efe26d7, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T15:14:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,582 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for f012bfcce8bd6aef12a49ca6ef302f96 in 1768ms, sequenceid=95, compaction requested=false 2024-11-19T15:14:16,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:16,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-19T15:14:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-19T15:14:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-19T15:14:16,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0850 sec 2024-11-19T15:14:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 4.0920 sec 2024-11-19T15:14:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T15:14:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,665 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,665 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-19T15:14:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T15:14:16,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,668 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:16,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,669 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:16,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:16,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T15:14:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T15:14:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:16,822 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T15:14:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:16,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:16,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:16,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:16,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:16,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a75ec5c20f934656ba5887faf5aa2a3e_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029255069/Put/seqid=0 2024-11-19T15:14:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741977_1153 (size=9714) 2024-11-19T15:14:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,862 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a75ec5c20f934656ba5887faf5aa2a3e_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a75ec5c20f934656ba5887faf5aa2a3e_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/38cf53432e9345b680416d8679ddf400, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/38cf53432e9345b680416d8679ddf400 is 175, key is test_row_0/A:col10/1732029255069/Put/seqid=0 2024-11-19T15:14:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741978_1154 (size=22361) 2024-11-19T15:14:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T15:14:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:17,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,275 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/38cf53432e9345b680416d8679ddf400 2024-11-19T15:14:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T15:14:17,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/4009ffdf99f14c39b1179e5a7f6cc0d5 is 50, key is test_row_0/B:col10/1732029255069/Put/seqid=0 2024-11-19T15:14:17,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741979_1155 (size=9657) 2024-11-19T15:14:17,311 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/4009ffdf99f14c39b1179e5a7f6cc0d5 2024-11-19T15:14:17,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/68003efe5b2940a5ba7e8859ce892796 is 50, key is test_row_0/C:col10/1732029255069/Put/seqid=0 2024-11-19T15:14:17,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741980_1156 (size=9657) 2024-11-19T15:14:17,349 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/68003efe5b2940a5ba7e8859ce892796 2024-11-19T15:14:17,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029257269, max timestamp=1732029257348Num uniques 859; 2024-11-19T15:14:17,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029257269, max timestamp=1732029257348Num uniques 859; 2024-11-19T15:14:17,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:17,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:17,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029257269, max timestamp=1732029257348Num uniques 859; 2024-11-19T15:14:17,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:14:17,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/38cf53432e9345b680416d8679ddf400 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/38cf53432e9345b680416d8679ddf400 2024-11-19T15:14:17,364 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/38cf53432e9345b680416d8679ddf400, entries=100, sequenceid=109, filesize=21.8 K 2024-11-19T15:14:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/4009ffdf99f14c39b1179e5a7f6cc0d5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/4009ffdf99f14c39b1179e5a7f6cc0d5 2024-11-19T15:14:17,374 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/4009ffdf99f14c39b1179e5a7f6cc0d5, entries=100, sequenceid=109, filesize=9.4 K 2024-11-19T15:14:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/68003efe5b2940a5ba7e8859ce892796 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/68003efe5b2940a5ba7e8859ce892796 2024-11-19T15:14:17,381 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/68003efe5b2940a5ba7e8859ce892796, entries=100, sequenceid=109, filesize=9.4 K 2024-11-19T15:14:17,383 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=64.85 KB/66410 for f012bfcce8bd6aef12a49ca6ef302f96 in 561ms, sequenceid=109, compaction requested=true 2024-11-19T15:14:17,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:17,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:17,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-19T15:14:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-19T15:14:17,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=71.56 KB heapSize=195.92 KB 2024-11-19T15:14:17,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-19T15:14:17,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:17,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 716 msec 2024-11-19T15:14:17,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:17,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:17,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:17,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:17,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:17,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 740 msec 2024-11-19T15:14:17,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fcd7208ae0f443e59a7ce668ef909ab2_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029257385/Put/seqid=0 2024-11-19T15:14:17,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741981_1157 (size=12304) 2024-11-19T15:14:17,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:17,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029317458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:17,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:17,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029317465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:17,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029317466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:17,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029317574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:17,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029317574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:17,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:17,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029317584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:17,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029317784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:17,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029317784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:17,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029317794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T15:14:17,805 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:17,805 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-19T15:14:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T15:14:17,807 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:17,808 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:17,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:17,831 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:17,836 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fcd7208ae0f443e59a7ce668ef909ab2_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fcd7208ae0f443e59a7ce668ef909ab2_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:17,837 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/1e10c31bfc8943bc98dd892352d5c747, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:17,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/1e10c31bfc8943bc98dd892352d5c747 is 175, key is test_row_0/A:col10/1732029257385/Put/seqid=0 2024-11-19T15:14:17,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741982_1158 (size=31105) 2024-11-19T15:14:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T15:14:17,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T15:14:17,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:17,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:17,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:17,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:17,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:18,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029318094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029318094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029318104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T15:14:18,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:18,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:18,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:18,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:18,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:18,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029318124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T15:14:18,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029318124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,125 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in f012bfcce8bd6aef12a49ca6ef302f96 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,125 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in f012bfcce8bd6aef12a49ca6ef302f96 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,243 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/1e10c31bfc8943bc98dd892352d5c747 2024-11-19T15:14:18,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/c4357cb516d04296a9a4e5f62ef18937 is 50, key is test_row_0/B:col10/1732029257385/Put/seqid=0 2024-11-19T15:14:18,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741983_1159 (size=12151) 2024-11-19T15:14:18,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/c4357cb516d04296a9a4e5f62ef18937 2024-11-19T15:14:18,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/41319eeca1804091810d23405c69d01c is 50, key is test_row_0/C:col10/1732029257385/Put/seqid=0 2024-11-19T15:14:18,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T15:14:18,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:18,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:18,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:18,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:18,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:18,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741984_1160 (size=12151) 2024-11-19T15:14:18,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/41319eeca1804091810d23405c69d01c 2024-11-19T15:14:18,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/1e10c31bfc8943bc98dd892352d5c747 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/1e10c31bfc8943bc98dd892352d5c747 2024-11-19T15:14:18,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/1e10c31bfc8943bc98dd892352d5c747, entries=150, sequenceid=136, filesize=30.4 K 2024-11-19T15:14:18,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/c4357cb516d04296a9a4e5f62ef18937 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c4357cb516d04296a9a4e5f62ef18937 2024-11-19T15:14:18,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c4357cb516d04296a9a4e5f62ef18937, entries=150, sequenceid=136, filesize=11.9 K 2024-11-19T15:14:18,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/41319eeca1804091810d23405c69d01c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/41319eeca1804091810d23405c69d01c 2024-11-19T15:14:18,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/41319eeca1804091810d23405c69d01c, entries=150, sequenceid=136, filesize=11.9 K 2024-11-19T15:14:18,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~195.88 KB/200576, currentSize=107.34 KB/109920 for f012bfcce8bd6aef12a49ca6ef302f96 in 919ms, sequenceid=136, compaction requested=true 2024-11-19T15:14:18,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:18,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:18,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:18,323 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:14:18,323 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:14:18,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:18,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:18,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:18,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:18,325 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:14:18,325 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 115479 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:14:18,325 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/A is initiating minor compaction (all files) 2024-11-19T15:14:18,325 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/B is initiating minor compaction (all files) 2024-11-19T15:14:18,325 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/A in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:18,325 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c7569166fd5a4673bedbe6cee0e170a1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2662366243764452b21379ffdfedf2b5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/38cf53432e9345b680416d8679ddf400, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/1e10c31bfc8943bc98dd892352d5c747] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=112.8 K 2024-11-19T15:14:18,325 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:18,325 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c7569166fd5a4673bedbe6cee0e170a1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2662366243764452b21379ffdfedf2b5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/38cf53432e9345b680416d8679ddf400, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/1e10c31bfc8943bc98dd892352d5c747] 2024-11-19T15:14:18,325 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/B in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:18,326 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/3c5b318be2364ded9d4335d30d15fde4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/d88fc8e0d89948c2802f66fd46883d9c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/4009ffdf99f14c39b1179e5a7f6cc0d5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c4357cb516d04296a9a4e5f62ef18937] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=44.8 K 2024-11-19T15:14:18,326 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7569166fd5a4673bedbe6cee0e170a1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029253241 2024-11-19T15:14:18,326 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c5b318be2364ded9d4335d30d15fde4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029253241 2024-11-19T15:14:18,326 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2662366243764452b21379ffdfedf2b5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732029253931 2024-11-19T15:14:18,327 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d88fc8e0d89948c2802f66fd46883d9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732029253931 2024-11-19T15:14:18,329 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 4009ffdf99f14c39b1179e5a7f6cc0d5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732029255064 2024-11-19T15:14:18,329 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38cf53432e9345b680416d8679ddf400, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732029255064 2024-11-19T15:14:18,329 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting c4357cb516d04296a9a4e5f62ef18937, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732029257368 2024-11-19T15:14:18,332 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e10c31bfc8943bc98dd892352d5c747, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732029257368 2024-11-19T15:14:18,350 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:18,350 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/f18fb8a7f2044945a0b7b03809518b1f is 50, key is test_row_0/B:col10/1732029257385/Put/seqid=0 2024-11-19T15:14:18,351 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:18,354 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411196b47c83995aa4349a384b21741247e86_f012bfcce8bd6aef12a49ca6ef302f96 store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:18,357 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411196b47c83995aa4349a384b21741247e86_f012bfcce8bd6aef12a49ca6ef302f96, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:18,357 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196b47c83995aa4349a384b21741247e86_f012bfcce8bd6aef12a49ca6ef302f96 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:18,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741985_1161 (size=12391) 2024-11-19T15:14:18,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741986_1162 (size=4469) 2024-11-19T15:14:18,380 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#A#compaction#139 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:18,380 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/0d293598b3a547bc96e4f6cb33e01c11 is 175, key is test_row_0/A:col10/1732029257385/Put/seqid=0 2024-11-19T15:14:18,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741987_1163 (size=31345) 2024-11-19T15:14:18,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T15:14:18,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:18,422 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T15:14:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T15:14:18,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195134c3a51d774f12850f815f06b43c25_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029257465/Put/seqid=0 2024-11-19T15:14:18,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741988_1164 (size=12304) 2024-11-19T15:14:18,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:18,457 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195134c3a51d774f12850f815f06b43c25_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195134c3a51d774f12850f815f06b43c25_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:18,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/4a18c1af4d8f496e89f929c8c4e2a07a, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:18,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/4a18c1af4d8f496e89f929c8c4e2a07a is 175, key is test_row_0/A:col10/1732029257465/Put/seqid=0 2024-11-19T15:14:18,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741989_1165 (size=31105) 2024-11-19T15:14:18,470 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/4a18c1af4d8f496e89f929c8c4e2a07a 2024-11-19T15:14:18,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/6ed132b1f12d465dba35238b1dfeb495 is 50, key is test_row_0/B:col10/1732029257465/Put/seqid=0 2024-11-19T15:14:18,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741990_1166 (size=12151) 2024-11-19T15:14:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:18,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:18,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029318634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029318634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029318635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029318754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029318754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029318754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,777 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/f18fb8a7f2044945a0b7b03809518b1f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/f18fb8a7f2044945a0b7b03809518b1f 2024-11-19T15:14:18,783 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/B of f012bfcce8bd6aef12a49ca6ef302f96 into f18fb8a7f2044945a0b7b03809518b1f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:18,783 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:18,783 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/B, priority=12, startTime=1732029258323; duration=0sec 2024-11-19T15:14:18,783 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:18,783 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:B 2024-11-19T15:14:18,784 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:14:18,785 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:14:18,785 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/C is initiating minor compaction (all files) 2024-11-19T15:14:18,785 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/C in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:18,786 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/6da625c8ae184472859d96d9de39d321, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/f84bc52d160b4e7fa9c57a3c7efe26d7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/68003efe5b2940a5ba7e8859ce892796, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/41319eeca1804091810d23405c69d01c] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=44.8 K 2024-11-19T15:14:18,786 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 6da625c8ae184472859d96d9de39d321, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029253241 2024-11-19T15:14:18,786 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting f84bc52d160b4e7fa9c57a3c7efe26d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732029253931 2024-11-19T15:14:18,787 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 68003efe5b2940a5ba7e8859ce892796, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732029255064 2024-11-19T15:14:18,787 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 41319eeca1804091810d23405c69d01c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732029257368 2024-11-19T15:14:18,799 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#C#compaction#142 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:18,800 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/0869ae7cf21c48c3b7fe6d55924153d4 is 50, key is test_row_0/C:col10/1732029257385/Put/seqid=0 2024-11-19T15:14:18,805 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/0d293598b3a547bc96e4f6cb33e01c11 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/0d293598b3a547bc96e4f6cb33e01c11 2024-11-19T15:14:18,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741991_1167 (size=12391) 2024-11-19T15:14:18,815 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/A of f012bfcce8bd6aef12a49ca6ef302f96 into 0d293598b3a547bc96e4f6cb33e01c11(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:18,815 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:18,815 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/A, priority=12, startTime=1732029258323; duration=0sec 2024-11-19T15:14:18,815 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:18,815 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:A 2024-11-19T15:14:18,895 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/6ed132b1f12d465dba35238b1dfeb495 2024-11-19T15:14:18,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/e160f099f6714af1b6319ef5785ebcbb is 50, key is test_row_0/C:col10/1732029257465/Put/seqid=0 2024-11-19T15:14:18,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741992_1168 (size=12151) 2024-11-19T15:14:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T15:14:18,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029318964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029318964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029318964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:19,212 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/0869ae7cf21c48c3b7fe6d55924153d4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/0869ae7cf21c48c3b7fe6d55924153d4 2024-11-19T15:14:19,217 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/C of f012bfcce8bd6aef12a49ca6ef302f96 into 0869ae7cf21c48c3b7fe6d55924153d4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:19,217 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:19,217 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/C, priority=12, startTime=1732029258323; duration=0sec 2024-11-19T15:14:19,217 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:19,217 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:C 2024-11-19T15:14:19,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:19,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029319274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029319274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:19,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029319274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:19,310 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/e160f099f6714af1b6319ef5785ebcbb 2024-11-19T15:14:19,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/4a18c1af4d8f496e89f929c8c4e2a07a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/4a18c1af4d8f496e89f929c8c4e2a07a 2024-11-19T15:14:19,320 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/4a18c1af4d8f496e89f929c8c4e2a07a, entries=150, sequenceid=155, filesize=30.4 K 2024-11-19T15:14:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/6ed132b1f12d465dba35238b1dfeb495 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/6ed132b1f12d465dba35238b1dfeb495 2024-11-19T15:14:19,327 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/6ed132b1f12d465dba35238b1dfeb495, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T15:14:19,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/e160f099f6714af1b6319ef5785ebcbb as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/e160f099f6714af1b6319ef5785ebcbb 2024-11-19T15:14:19,332 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/e160f099f6714af1b6319ef5785ebcbb, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T15:14:19,333 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f012bfcce8bd6aef12a49ca6ef302f96 in 912ms, sequenceid=155, compaction requested=false 2024-11-19T15:14:19,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:19,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:19,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-19T15:14:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-19T15:14:19,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-19T15:14:19,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5270 sec 2024-11-19T15:14:19,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5320 sec 2024-11-19T15:14:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:19,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T15:14:19,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:19,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:19,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:19,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:19,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:19,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:19,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bb447704a55947398bc9729aa3cf7bea_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029259787/Put/seqid=0 2024-11-19T15:14:19,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741993_1169 (size=17284) 2024-11-19T15:14:19,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029319819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:19,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029319820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:19,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029319820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:19,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029319924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:19,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029319934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:19,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029319934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T15:14:19,955 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:19,955 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-19T15:14:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T15:14:19,957 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:19,958 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:19,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T15:14:20,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T15:14:20,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:20,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:20,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:20,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029320134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029320144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029320144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:20,206 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:20,210 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bb447704a55947398bc9729aa3cf7bea_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bb447704a55947398bc9729aa3cf7bea_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:20,211 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/26187bbd77674b38bfddd048ae64aaa0, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:20,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/26187bbd77674b38bfddd048ae64aaa0 is 175, key is test_row_0/A:col10/1732029259787/Put/seqid=0 2024-11-19T15:14:20,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741994_1170 (size=48389) 2024-11-19T15:14:20,217 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/26187bbd77674b38bfddd048ae64aaa0 2024-11-19T15:14:20,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/512d9a2ec3de4bf68d1d572559f09d8b is 50, key is test_row_0/B:col10/1732029259787/Put/seqid=0 2024-11-19T15:14:20,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741995_1171 (size=12151) 2024-11-19T15:14:20,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T15:14:20,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:20,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:20,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:20,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T15:14:20,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T15:14:20,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:20,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:20,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:20,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029320444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:20,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029320454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:20,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029320454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:20,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T15:14:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:20,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T15:14:20,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/512d9a2ec3de4bf68d1d572559f09d8b 2024-11-19T15:14:20,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/1d0b3c1fd3f2435e91c07b68d1e98618 is 50, key is test_row_0/C:col10/1732029259787/Put/seqid=0 2024-11-19T15:14:20,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741996_1172 (size=12151) 2024-11-19T15:14:20,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T15:14:20,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:20,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:20,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:20,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T15:14:20,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:20,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:20,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:20,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:20,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029320954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:20,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029320964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:20,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029320964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:20,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:21,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T15:14:21,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:21,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:21,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:21,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:21,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:21,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/1d0b3c1fd3f2435e91c07b68d1e98618 2024-11-19T15:14:21,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/26187bbd77674b38bfddd048ae64aaa0 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/26187bbd77674b38bfddd048ae64aaa0 2024-11-19T15:14:21,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/26187bbd77674b38bfddd048ae64aaa0, entries=250, sequenceid=173, filesize=47.3 K 2024-11-19T15:14:21,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/512d9a2ec3de4bf68d1d572559f09d8b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/512d9a2ec3de4bf68d1d572559f09d8b 2024-11-19T15:14:21,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/512d9a2ec3de4bf68d1d572559f09d8b, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T15:14:21,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/1d0b3c1fd3f2435e91c07b68d1e98618 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/1d0b3c1fd3f2435e91c07b68d1e98618 2024-11-19T15:14:21,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/1d0b3c1fd3f2435e91c07b68d1e98618, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T15:14:21,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f012bfcce8bd6aef12a49ca6ef302f96 in 1281ms, sequenceid=173, compaction requested=true 2024-11-19T15:14:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:21,068 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:21,068 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:21,070 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:21,070 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:21,070 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/B is initiating minor compaction (all files) 2024-11-19T15:14:21,070 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/A is initiating minor compaction (all files) 2024-11-19T15:14:21,070 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/B in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:21,070 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/A in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:21,070 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/f18fb8a7f2044945a0b7b03809518b1f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/6ed132b1f12d465dba35238b1dfeb495, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/512d9a2ec3de4bf68d1d572559f09d8b] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=35.8 K 2024-11-19T15:14:21,070 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/0d293598b3a547bc96e4f6cb33e01c11, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/4a18c1af4d8f496e89f929c8c4e2a07a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/26187bbd77674b38bfddd048ae64aaa0] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=108.2 K 2024-11-19T15:14:21,070 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:21,070 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/0d293598b3a547bc96e4f6cb33e01c11, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/4a18c1af4d8f496e89f929c8c4e2a07a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/26187bbd77674b38bfddd048ae64aaa0] 2024-11-19T15:14:21,070 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting f18fb8a7f2044945a0b7b03809518b1f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732029257368 2024-11-19T15:14:21,071 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d293598b3a547bc96e4f6cb33e01c11, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732029257368 2024-11-19T15:14:21,071 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ed132b1f12d465dba35238b1dfeb495, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732029257457 2024-11-19T15:14:21,071 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a18c1af4d8f496e89f929c8c4e2a07a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732029257457 2024-11-19T15:14:21,071 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 512d9a2ec3de4bf68d1d572559f09d8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732029258626 2024-11-19T15:14:21,071 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26187bbd77674b38bfddd048ae64aaa0, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732029258626 2024-11-19T15:14:21,079 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:21,082 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119436f4011b9b84892913f9000d3d16ca4_f012bfcce8bd6aef12a49ca6ef302f96 store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:21,082 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#B#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:21,083 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/c4185c90751d4f948d8c02e97373ee51 is 50, key is test_row_0/B:col10/1732029259787/Put/seqid=0 2024-11-19T15:14:21,084 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119436f4011b9b84892913f9000d3d16ca4_f012bfcce8bd6aef12a49ca6ef302f96, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:21,085 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119436f4011b9b84892913f9000d3d16ca4_f012bfcce8bd6aef12a49ca6ef302f96 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T15:14:21,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741997_1173 (size=12493) 2024-11-19T15:14:21,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741998_1174 (size=4469) 2024-11-19T15:14:21,127 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#A#compaction#147 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:21,128 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/e1c94128e0da405fa84fed22c81c0e88 is 175, key is test_row_0/A:col10/1732029259787/Put/seqid=0 2024-11-19T15:14:21,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741999_1175 (size=31447) 2024-11-19T15:14:21,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T15:14:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:21,185 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T15:14:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b80e908694c44c5490a9ea9a7653f0e7_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029259817/Put/seqid=0 2024-11-19T15:14:21,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742000_1176 (size=12304) 2024-11-19T15:14:21,504 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/c4185c90751d4f948d8c02e97373ee51 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c4185c90751d4f948d8c02e97373ee51 2024-11-19T15:14:21,511 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/B of f012bfcce8bd6aef12a49ca6ef302f96 into c4185c90751d4f948d8c02e97373ee51(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:21,511 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:21,511 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/B, priority=13, startTime=1732029261068; duration=0sec 2024-11-19T15:14:21,511 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:21,511 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:B 2024-11-19T15:14:21,511 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:21,513 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:21,513 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/C is initiating minor compaction (all files) 2024-11-19T15:14:21,513 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/C in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:21,514 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/0869ae7cf21c48c3b7fe6d55924153d4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/e160f099f6714af1b6319ef5785ebcbb, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/1d0b3c1fd3f2435e91c07b68d1e98618] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=35.8 K 2024-11-19T15:14:21,517 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 0869ae7cf21c48c3b7fe6d55924153d4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732029257368 2024-11-19T15:14:21,517 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting e160f099f6714af1b6319ef5785ebcbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732029257457 2024-11-19T15:14:21,518 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d0b3c1fd3f2435e91c07b68d1e98618, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732029258626 2024-11-19T15:14:21,547 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/e1c94128e0da405fa84fed22c81c0e88 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e1c94128e0da405fa84fed22c81c0e88 2024-11-19T15:14:21,556 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/A of f012bfcce8bd6aef12a49ca6ef302f96 into e1c94128e0da405fa84fed22c81c0e88(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:21,556 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:21,556 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/A, priority=13, startTime=1732029261068; duration=0sec 2024-11-19T15:14:21,556 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:21,557 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:A 2024-11-19T15:14:21,557 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:21,558 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/82748c186c404062adbf47f532428554 is 50, key is test_row_0/C:col10/1732029259787/Put/seqid=0 2024-11-19T15:14:21,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742001_1177 (size=12493) 2024-11-19T15:14:21,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:21,600 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/82748c186c404062adbf47f532428554 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/82748c186c404062adbf47f532428554 2024-11-19T15:14:21,607 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b80e908694c44c5490a9ea9a7653f0e7_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b80e908694c44c5490a9ea9a7653f0e7_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:21,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/96b42e41b71349b697addffb412af8e8, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:21,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/96b42e41b71349b697addffb412af8e8 is 175, key is test_row_0/A:col10/1732029259817/Put/seqid=0 2024-11-19T15:14:21,612 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/C of f012bfcce8bd6aef12a49ca6ef302f96 into 82748c186c404062adbf47f532428554(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:21,612 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:21,612 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/C, priority=13, startTime=1732029261068; duration=0sec 2024-11-19T15:14:21,612 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:21,612 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:C 2024-11-19T15:14:21,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742002_1178 (size=31105) 2024-11-19T15:14:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:21,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:22,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029321996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029321997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029321997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,020 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/96b42e41b71349b697addffb412af8e8 2024-11-19T15:14:22,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/717da90e65b4467b8de627f37bca4faf is 50, key is test_row_0/B:col10/1732029259817/Put/seqid=0 2024-11-19T15:14:22,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742003_1179 (size=12151) 2024-11-19T15:14:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T15:14:22,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029322104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029322115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029322115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029322144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,152 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in f012bfcce8bd6aef12a49ca6ef302f96 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:22,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029322164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,170 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in f012bfcce8bd6aef12a49ca6ef302f96 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029322314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029322324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029322324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,442 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/717da90e65b4467b8de627f37bca4faf 2024-11-19T15:14:22,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/55668fef110844e5932f82483b8b13a1 is 50, key is test_row_0/C:col10/1732029259817/Put/seqid=0 2024-11-19T15:14:22,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742004_1180 (size=12151) 2024-11-19T15:14:22,481 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/55668fef110844e5932f82483b8b13a1 2024-11-19T15:14:22,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/96b42e41b71349b697addffb412af8e8 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/96b42e41b71349b697addffb412af8e8 2024-11-19T15:14:22,493 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/96b42e41b71349b697addffb412af8e8, entries=150, sequenceid=190, filesize=30.4 K 2024-11-19T15:14:22,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/717da90e65b4467b8de627f37bca4faf as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/717da90e65b4467b8de627f37bca4faf 2024-11-19T15:14:22,503 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/717da90e65b4467b8de627f37bca4faf, entries=150, sequenceid=190, filesize=11.9 K 2024-11-19T15:14:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/55668fef110844e5932f82483b8b13a1 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/55668fef110844e5932f82483b8b13a1 2024-11-19T15:14:22,510 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/55668fef110844e5932f82483b8b13a1, entries=150, sequenceid=190, filesize=11.9 K 2024-11-19T15:14:22,512 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for f012bfcce8bd6aef12a49ca6ef302f96 in 1328ms, sequenceid=190, compaction requested=false 2024-11-19T15:14:22,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:22,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:22,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-19T15:14:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-19T15:14:22,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-19T15:14:22,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5560 sec 2024-11-19T15:14:22,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.5620 sec 2024-11-19T15:14:22,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T15:14:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:22,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:22,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:22,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:22,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:22,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:22,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:22,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195db420ce8bfc44218685516429df5672_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029262626/Put/seqid=0 2024-11-19T15:14:22,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029322666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029322668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029322670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742005_1181 (size=17284) 2024-11-19T15:14:22,686 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:22,692 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195db420ce8bfc44218685516429df5672_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195db420ce8bfc44218685516429df5672_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:22,694 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/895010eb07034f2d8c08cb0c833edc87, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:22,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/895010eb07034f2d8c08cb0c833edc87 is 175, key is test_row_0/A:col10/1732029262626/Put/seqid=0 2024-11-19T15:14:22,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742006_1182 (size=48389) 2024-11-19T15:14:22,712 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/895010eb07034f2d8c08cb0c833edc87 2024-11-19T15:14:22,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/eab65e31bf8444aa94af5d4379b83826 is 50, key is test_row_0/B:col10/1732029262626/Put/seqid=0 2024-11-19T15:14:22,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742007_1183 (size=12151) 2024-11-19T15:14:22,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029322784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029322785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029322785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029323004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029323004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029323004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/eab65e31bf8444aa94af5d4379b83826 2024-11-19T15:14:23,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/56e34391c65c4e4abbabf581dd9c342d is 50, key is test_row_0/C:col10/1732029262626/Put/seqid=0 2024-11-19T15:14:23,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742008_1184 (size=12151) 2024-11-19T15:14:23,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/56e34391c65c4e4abbabf581dd9c342d 2024-11-19T15:14:23,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/895010eb07034f2d8c08cb0c833edc87 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/895010eb07034f2d8c08cb0c833edc87 2024-11-19T15:14:23,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/895010eb07034f2d8c08cb0c833edc87, entries=250, sequenceid=211, filesize=47.3 K 2024-11-19T15:14:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/eab65e31bf8444aa94af5d4379b83826 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/eab65e31bf8444aa94af5d4379b83826 2024-11-19T15:14:23,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/eab65e31bf8444aa94af5d4379b83826, entries=150, sequenceid=211, filesize=11.9 K 2024-11-19T15:14:23,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/56e34391c65c4e4abbabf581dd9c342d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/56e34391c65c4e4abbabf581dd9c342d 2024-11-19T15:14:23,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/56e34391c65c4e4abbabf581dd9c342d, entries=150, sequenceid=211, filesize=11.9 K 2024-11-19T15:14:23,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f012bfcce8bd6aef12a49ca6ef302f96 in 594ms, sequenceid=211, compaction requested=true 2024-11-19T15:14:23,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:23,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:23,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:23,235 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:23,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:23,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:23,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:23,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T15:14:23,236 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:23,238 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:23,238 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/A is initiating minor compaction (all files) 2024-11-19T15:14:23,238 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/A in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:23,239 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e1c94128e0da405fa84fed22c81c0e88, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/96b42e41b71349b697addffb412af8e8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/895010eb07034f2d8c08cb0c833edc87] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=108.3 K 2024-11-19T15:14:23,239 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:23,239 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e1c94128e0da405fa84fed22c81c0e88, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/96b42e41b71349b697addffb412af8e8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/895010eb07034f2d8c08cb0c833edc87] 2024-11-19T15:14:23,239 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:23,239 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/B is initiating minor compaction (all files) 2024-11-19T15:14:23,239 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/B in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:23,240 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c4185c90751d4f948d8c02e97373ee51, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/717da90e65b4467b8de627f37bca4faf, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/eab65e31bf8444aa94af5d4379b83826] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=35.9 K 2024-11-19T15:14:23,240 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1c94128e0da405fa84fed22c81c0e88, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732029258626 2024-11-19T15:14:23,240 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting c4185c90751d4f948d8c02e97373ee51, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732029258626 2024-11-19T15:14:23,241 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 717da90e65b4467b8de627f37bca4faf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732029259817 2024-11-19T15:14:23,241 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b42e41b71349b697addffb412af8e8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732029259817 2024-11-19T15:14:23,243 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting eab65e31bf8444aa94af5d4379b83826, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732029261996 2024-11-19T15:14:23,243 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 895010eb07034f2d8c08cb0c833edc87, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732029261989 2024-11-19T15:14:23,262 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:23,263 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/622822b6f06c4d70af342e19cd1f4557 is 50, key is test_row_0/B:col10/1732029262626/Put/seqid=0 2024-11-19T15:14:23,281 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:23,305 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411198d5abf4aea4742d686aaf15677649d6c_f012bfcce8bd6aef12a49ca6ef302f96 store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:23,308 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411198d5abf4aea4742d686aaf15677649d6c_f012bfcce8bd6aef12a49ca6ef302f96, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:23,308 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198d5abf4aea4742d686aaf15677649d6c_f012bfcce8bd6aef12a49ca6ef302f96 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:23,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T15:14:23,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:23,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:23,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:23,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:23,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:23,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:23,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742009_1185 (size=12595) 2024-11-19T15:14:23,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029323350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029323351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029323353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742010_1186 (size=4469) 2024-11-19T15:14:23,367 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#A#compaction#157 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:23,368 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/9d2bef2b22ea4b26a57ac9dadd2f93b3 is 175, key is test_row_0/A:col10/1732029262626/Put/seqid=0 2024-11-19T15:14:23,374 INFO [master/50570b5b74e7:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T15:14:23,374 INFO [master/50570b5b74e7:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T15:14:23,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111989dcc44bf88d44a2a135246009661bf4_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029262662/Put/seqid=0 2024-11-19T15:14:23,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742011_1187 (size=31549) 2024-11-19T15:14:23,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742012_1188 (size=17284) 2024-11-19T15:14:23,460 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:23,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029323465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,472 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111989dcc44bf88d44a2a135246009661bf4_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111989dcc44bf88d44a2a135246009661bf4_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:23,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029323465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029323465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,479 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/2c18a8365b0848e9906c05b2b942969b, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:23,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/2c18a8365b0848e9906c05b2b942969b is 175, key is test_row_0/A:col10/1732029262662/Put/seqid=0 2024-11-19T15:14:23,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742013_1189 (size=48389) 2024-11-19T15:14:23,531 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/2c18a8365b0848e9906c05b2b942969b 2024-11-19T15:14:23,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/3249eeb2d6784009a96a9bae0a14ecbd is 50, key is test_row_0/B:col10/1732029262662/Put/seqid=0 2024-11-19T15:14:23,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742014_1190 (size=12151) 2024-11-19T15:14:23,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/3249eeb2d6784009a96a9bae0a14ecbd 2024-11-19T15:14:23,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/35ef696c2826446bbac5cd36165cc7c3 is 50, key is test_row_0/C:col10/1732029262662/Put/seqid=0 2024-11-19T15:14:23,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742015_1191 (size=12151) 2024-11-19T15:14:23,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029323684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029323684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029323684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,737 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/622822b6f06c4d70af342e19cd1f4557 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/622822b6f06c4d70af342e19cd1f4557 2024-11-19T15:14:23,747 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/B of f012bfcce8bd6aef12a49ca6ef302f96 into 622822b6f06c4d70af342e19cd1f4557(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:23,747 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:23,747 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/B, priority=13, startTime=1732029263235; duration=0sec 2024-11-19T15:14:23,747 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:23,747 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:B 2024-11-19T15:14:23,747 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:23,748 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:23,748 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/C is initiating minor compaction (all files) 2024-11-19T15:14:23,748 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/C in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:23,748 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/82748c186c404062adbf47f532428554, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/55668fef110844e5932f82483b8b13a1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/56e34391c65c4e4abbabf581dd9c342d] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=35.9 K 2024-11-19T15:14:23,752 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 82748c186c404062adbf47f532428554, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732029258626 2024-11-19T15:14:23,753 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 55668fef110844e5932f82483b8b13a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732029259817 2024-11-19T15:14:23,753 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 56e34391c65c4e4abbabf581dd9c342d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732029261996 2024-11-19T15:14:23,761 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#C#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:23,761 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/2c252e96df514a898a643d9d26629f33 is 50, key is test_row_0/C:col10/1732029262626/Put/seqid=0 2024-11-19T15:14:23,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742016_1192 (size=12595) 2024-11-19T15:14:23,846 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/9d2bef2b22ea4b26a57ac9dadd2f93b3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/9d2bef2b22ea4b26a57ac9dadd2f93b3 2024-11-19T15:14:23,855 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/A of f012bfcce8bd6aef12a49ca6ef302f96 into 9d2bef2b22ea4b26a57ac9dadd2f93b3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:23,856 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:23,856 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/A, priority=13, startTime=1732029263235; duration=0sec 2024-11-19T15:14:23,856 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:23,856 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:A 2024-11-19T15:14:23,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029323994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:23,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029323994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:24,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029324004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:24,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/35ef696c2826446bbac5cd36165cc7c3 2024-11-19T15:14:24,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/2c18a8365b0848e9906c05b2b942969b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2c18a8365b0848e9906c05b2b942969b 2024-11-19T15:14:24,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2c18a8365b0848e9906c05b2b942969b, entries=250, sequenceid=227, filesize=47.3 K 2024-11-19T15:14:24,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/3249eeb2d6784009a96a9bae0a14ecbd as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/3249eeb2d6784009a96a9bae0a14ecbd 2024-11-19T15:14:24,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/3249eeb2d6784009a96a9bae0a14ecbd, entries=150, sequenceid=227, filesize=11.9 K 2024-11-19T15:14:24,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/35ef696c2826446bbac5cd36165cc7c3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/35ef696c2826446bbac5cd36165cc7c3 2024-11-19T15:14:24,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/35ef696c2826446bbac5cd36165cc7c3, entries=150, sequenceid=227, filesize=11.9 K 2024-11-19T15:14:24,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f012bfcce8bd6aef12a49ca6ef302f96 in 765ms, sequenceid=227, compaction requested=false 2024-11-19T15:14:24,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T15:14:24,134 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:24,135 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-19T15:14:24,137 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:24,138 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:24,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T15:14:24,199 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/2c252e96df514a898a643d9d26629f33 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/2c252e96df514a898a643d9d26629f33 2024-11-19T15:14:24,208 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/C of f012bfcce8bd6aef12a49ca6ef302f96 into 2c252e96df514a898a643d9d26629f33(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:24,208 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:24,208 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/C, priority=13, startTime=1732029263235; duration=0sec 2024-11-19T15:14:24,208 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:24,209 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:C 2024-11-19T15:14:24,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T15:14:24,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T15:14:24,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:24,291 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T15:14:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:24,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c738766cf2544c2f95450579b6ba9adf_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029263335/Put/seqid=0 2024-11-19T15:14:24,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742017_1193 (size=12304) 2024-11-19T15:14:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T15:14:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:24,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:24,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029324533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:24,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029324542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:24,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029324542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:24,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029324644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:24,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029324654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:24,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:24,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029324656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:24,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:24,724 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c738766cf2544c2f95450579b6ba9adf_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c738766cf2544c2f95450579b6ba9adf_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:24,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/830e849b742a4a72b7beb911ef26b042, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/830e849b742a4a72b7beb911ef26b042 is 175, key is test_row_0/A:col10/1732029263335/Put/seqid=0 2024-11-19T15:14:24,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742018_1194 (size=31105) 2024-11-19T15:14:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T15:14:24,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029324855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:24,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029324864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:24,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029324864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:24,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:25,137 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/830e849b742a4a72b7beb911ef26b042 2024-11-19T15:14:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/4271f555e0d5445c8499a58a8c535e2c is 50, key is test_row_0/B:col10/1732029263335/Put/seqid=0 2024-11-19T15:14:25,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029325164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:25,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742019_1195 (size=12151) 2024-11-19T15:14:25,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029325174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:25,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029325184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:25,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:25,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:25,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T15:14:25,575 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/4271f555e0d5445c8499a58a8c535e2c 2024-11-19T15:14:25,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/e7a3f140926046bb9294e48b509308a1 is 50, key is test_row_0/C:col10/1732029263335/Put/seqid=0 2024-11-19T15:14:25,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742020_1196 (size=12151) 2024-11-19T15:14:25,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029325685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:25,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:25,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:25,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029325685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:25,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:25,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029325694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:26,002 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/e7a3f140926046bb9294e48b509308a1 2024-11-19T15:14:26,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/830e849b742a4a72b7beb911ef26b042 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/830e849b742a4a72b7beb911ef26b042 2024-11-19T15:14:26,013 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/830e849b742a4a72b7beb911ef26b042, entries=150, sequenceid=246, filesize=30.4 K 2024-11-19T15:14:26,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/4271f555e0d5445c8499a58a8c535e2c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/4271f555e0d5445c8499a58a8c535e2c 2024-11-19T15:14:26,019 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/4271f555e0d5445c8499a58a8c535e2c, entries=150, sequenceid=246, filesize=11.9 K 2024-11-19T15:14:26,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/e7a3f140926046bb9294e48b509308a1 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/e7a3f140926046bb9294e48b509308a1 2024-11-19T15:14:26,028 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/e7a3f140926046bb9294e48b509308a1, entries=150, sequenceid=246, filesize=11.9 K 2024-11-19T15:14:26,029 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for f012bfcce8bd6aef12a49ca6ef302f96 in 1738ms, sequenceid=246, compaction requested=true 2024-11-19T15:14:26,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:26,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:26,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T15:14:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T15:14:26,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-19T15:14:26,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8930 sec 2024-11-19T15:14:26,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.9010 sec 2024-11-19T15:14:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T15:14:26,284 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:26,285 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-19T15:14:26,288 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T15:14:26,289 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:26,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T15:14:26,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T15:14:26,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:26,443 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T15:14:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:26,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198fefa0c3e8f04af5b192885b94e35259_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029264531/Put/seqid=0 2024-11-19T15:14:26,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742021_1197 (size=12454) 2024-11-19T15:14:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T15:14:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:26,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:26,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:26,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029326733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029326736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:26,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029326740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:26,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:26,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:26,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029326854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:26,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:26,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:26,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:26,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029326854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:26,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:26,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029326855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:26,886 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198fefa0c3e8f04af5b192885b94e35259_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198fefa0c3e8f04af5b192885b94e35259_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:26,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/e807cca4b2994857ba8fc06a08adac9a, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:26,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/e807cca4b2994857ba8fc06a08adac9a is 175, key is test_row_0/A:col10/1732029264531/Put/seqid=0 2024-11-19T15:14:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T15:14:26,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742022_1198 (size=31255) 2024-11-19T15:14:26,915 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/e807cca4b2994857ba8fc06a08adac9a 2024-11-19T15:14:26,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/720026d741154ed28c51f98e05365350 is 50, key is test_row_0/B:col10/1732029264531/Put/seqid=0 2024-11-19T15:14:26,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742023_1199 (size=12301) 2024-11-19T15:14:26,946 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/720026d741154ed28c51f98e05365350 2024-11-19T15:14:26,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/2059e4f7c9a34cf8a75183e156a38e63 is 50, key is test_row_0/C:col10/1732029264531/Put/seqid=0 2024-11-19T15:14:26,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742024_1200 (size=12301) 2024-11-19T15:14:26,987 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/2059e4f7c9a34cf8a75183e156a38e63 2024-11-19T15:14:26,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/e807cca4b2994857ba8fc06a08adac9a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e807cca4b2994857ba8fc06a08adac9a 2024-11-19T15:14:27,021 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e807cca4b2994857ba8fc06a08adac9a, entries=150, sequenceid=263, filesize=30.5 K 2024-11-19T15:14:27,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/720026d741154ed28c51f98e05365350 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/720026d741154ed28c51f98e05365350 2024-11-19T15:14:27,032 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/720026d741154ed28c51f98e05365350, entries=150, sequenceid=263, filesize=12.0 K 2024-11-19T15:14:27,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/2059e4f7c9a34cf8a75183e156a38e63 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/2059e4f7c9a34cf8a75183e156a38e63 2024-11-19T15:14:27,045 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/2059e4f7c9a34cf8a75183e156a38e63, entries=150, sequenceid=263, filesize=12.0 K 2024-11-19T15:14:27,047 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f012bfcce8bd6aef12a49ca6ef302f96 in 604ms, sequenceid=263, compaction requested=true 2024-11-19T15:14:27,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:27,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:27,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T15:14:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T15:14:27,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T15:14:27,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 760 msec 2024-11-19T15:14:27,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 767 msec 2024-11-19T15:14:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:27,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T15:14:27,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:27,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:27,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:27,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:27,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:27,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:27,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e7d077fff9c9430c923b603778b82f4c_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029267067/Put/seqid=0 2024-11-19T15:14:27,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742025_1201 (size=14994) 2024-11-19T15:14:27,100 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:27,105 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e7d077fff9c9430c923b603778b82f4c_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e7d077fff9c9430c923b603778b82f4c_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:27,106 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/ed3b5e5dfa634c59b1665ba9783eaaec, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:27,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/ed3b5e5dfa634c59b1665ba9783eaaec is 175, key is test_row_0/A:col10/1732029267067/Put/seqid=0 2024-11-19T15:14:27,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029327110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029327111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029327112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742026_1202 (size=39949) 2024-11-19T15:14:27,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029327224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029327225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029327225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T15:14:27,428 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:27,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-19T15:14:27,431 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T15:14:27,434 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:27,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:27,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029327434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029327434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029327435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T15:14:27,550 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/ed3b5e5dfa634c59b1665ba9783eaaec 2024-11-19T15:14:27,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/ee1c2596649a4314843858d09248b1b3 is 50, key is test_row_0/B:col10/1732029267067/Put/seqid=0 2024-11-19T15:14:27,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742027_1203 (size=12301) 2024-11-19T15:14:27,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/ee1c2596649a4314843858d09248b1b3 2024-11-19T15:14:27,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T15:14:27,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:27,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:27,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:27,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:27,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:27,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/fca1774c97f649aab3e37792c20cf450 is 50, key is test_row_0/C:col10/1732029267067/Put/seqid=0 2024-11-19T15:14:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:27,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742028_1204 (size=12301) 2024-11-19T15:14:27,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/fca1774c97f649aab3e37792c20cf450 2024-11-19T15:14:27,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/ed3b5e5dfa634c59b1665ba9783eaaec as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ed3b5e5dfa634c59b1665ba9783eaaec 2024-11-19T15:14:27,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ed3b5e5dfa634c59b1665ba9783eaaec, entries=200, sequenceid=279, filesize=39.0 K 2024-11-19T15:14:27,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/ee1c2596649a4314843858d09248b1b3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/ee1c2596649a4314843858d09248b1b3 2024-11-19T15:14:27,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/ee1c2596649a4314843858d09248b1b3, entries=150, sequenceid=279, filesize=12.0 K 2024-11-19T15:14:27,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/fca1774c97f649aab3e37792c20cf450 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/fca1774c97f649aab3e37792c20cf450 2024-11-19T15:14:27,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/fca1774c97f649aab3e37792c20cf450, entries=150, sequenceid=279, filesize=12.0 K 2024-11-19T15:14:27,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f012bfcce8bd6aef12a49ca6ef302f96 in 566ms, sequenceid=279, compaction requested=true 2024-11-19T15:14:27,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:27,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:27,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:27,635 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T15:14:27,636 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T15:14:27,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:27,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:27,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:27,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:27,642 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61499 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T15:14:27,642 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/B is initiating minor compaction (all files) 2024-11-19T15:14:27,642 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 182247 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T15:14:27,642 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/B in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:27,642 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/A is initiating minor compaction (all files) 2024-11-19T15:14:27,642 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/A in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:27,642 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/622822b6f06c4d70af342e19cd1f4557, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/3249eeb2d6784009a96a9bae0a14ecbd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/4271f555e0d5445c8499a58a8c535e2c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/720026d741154ed28c51f98e05365350, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/ee1c2596649a4314843858d09248b1b3] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=60.1 K 2024-11-19T15:14:27,643 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/9d2bef2b22ea4b26a57ac9dadd2f93b3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2c18a8365b0848e9906c05b2b942969b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/830e849b742a4a72b7beb911ef26b042, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e807cca4b2994857ba8fc06a08adac9a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ed3b5e5dfa634c59b1665ba9783eaaec] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=178.0 K 2024-11-19T15:14:27,643 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:27,643 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/9d2bef2b22ea4b26a57ac9dadd2f93b3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2c18a8365b0848e9906c05b2b942969b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/830e849b742a4a72b7beb911ef26b042, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e807cca4b2994857ba8fc06a08adac9a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ed3b5e5dfa634c59b1665ba9783eaaec] 2024-11-19T15:14:27,643 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 622822b6f06c4d70af342e19cd1f4557, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732029261996 2024-11-19T15:14:27,643 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d2bef2b22ea4b26a57ac9dadd2f93b3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732029261996 2024-11-19T15:14:27,644 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 3249eeb2d6784009a96a9bae0a14ecbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732029262662 2024-11-19T15:14:27,645 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c18a8365b0848e9906c05b2b942969b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732029262662 2024-11-19T15:14:27,646 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 4271f555e0d5445c8499a58a8c535e2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732029263335 2024-11-19T15:14:27,646 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 830e849b742a4a72b7beb911ef26b042, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732029263335 2024-11-19T15:14:27,646 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 720026d741154ed28c51f98e05365350, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732029264531 2024-11-19T15:14:27,646 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e807cca4b2994857ba8fc06a08adac9a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732029264531 2024-11-19T15:14:27,647 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting ee1c2596649a4314843858d09248b1b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732029266723 2024-11-19T15:14:27,647 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed3b5e5dfa634c59b1665ba9783eaaec, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732029266723 2024-11-19T15:14:27,671 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:27,672 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#B#compaction#171 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:27,673 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/2103259ef9e74851984da34661b9a4d5 is 50, key is test_row_0/B:col10/1732029267067/Put/seqid=0 2024-11-19T15:14:27,683 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119530408d2525d438d951c798eeb4000e3_f012bfcce8bd6aef12a49ca6ef302f96 store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:27,687 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119530408d2525d438d951c798eeb4000e3_f012bfcce8bd6aef12a49ca6ef302f96, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:27,687 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119530408d2525d438d951c798eeb4000e3_f012bfcce8bd6aef12a49ca6ef302f96 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:27,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742029_1205 (size=12915) 2024-11-19T15:14:27,698 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/2103259ef9e74851984da34661b9a4d5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/2103259ef9e74851984da34661b9a4d5 2024-11-19T15:14:27,706 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/B of f012bfcce8bd6aef12a49ca6ef302f96 into 2103259ef9e74851984da34661b9a4d5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:27,706 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:27,706 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/B, priority=11, startTime=1732029267635; duration=0sec 2024-11-19T15:14:27,707 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:27,707 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:B 2024-11-19T15:14:27,707 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T15:14:27,709 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61499 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T15:14:27,709 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/C is initiating minor compaction (all files) 2024-11-19T15:14:27,709 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/C in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:27,709 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/2c252e96df514a898a643d9d26629f33, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/35ef696c2826446bbac5cd36165cc7c3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/e7a3f140926046bb9294e48b509308a1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/2059e4f7c9a34cf8a75183e156a38e63, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/fca1774c97f649aab3e37792c20cf450] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=60.1 K 2024-11-19T15:14:27,710 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c252e96df514a898a643d9d26629f33, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732029261996 2024-11-19T15:14:27,710 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 35ef696c2826446bbac5cd36165cc7c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732029262662 2024-11-19T15:14:27,711 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting e7a3f140926046bb9294e48b509308a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732029263335 2024-11-19T15:14:27,711 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 2059e4f7c9a34cf8a75183e156a38e63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732029264531 2024-11-19T15:14:27,712 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting fca1774c97f649aab3e37792c20cf450, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732029266723 2024-11-19T15:14:27,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742030_1206 (size=4469) 2024-11-19T15:14:27,740 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#C#compaction#173 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:27,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T15:14:27,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:27,742 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T15:14:27,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:27,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:27,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:27,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:27,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:27,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T15:14:27,745 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/fe0b21c4529540a8b62991155da87970 is 50, key is test_row_0/C:col10/1732029267067/Put/seqid=0 2024-11-19T15:14:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:27,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:27,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742031_1207 (size=12915) 2024-11-19T15:14:27,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190a02ec29f99b4f77ad8b6ee0e4229596_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029267094/Put/seqid=0 2024-11-19T15:14:27,777 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/fe0b21c4529540a8b62991155da87970 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/fe0b21c4529540a8b62991155da87970 2024-11-19T15:14:27,788 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/C of f012bfcce8bd6aef12a49ca6ef302f96 into fe0b21c4529540a8b62991155da87970(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:27,788 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:27,788 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/C, priority=11, startTime=1732029267636; duration=0sec 2024-11-19T15:14:27,788 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:27,788 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:C 2024-11-19T15:14:27,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742032_1208 (size=12454) 2024-11-19T15:14:27,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:27,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029327790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029327801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029327802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,808 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190a02ec29f99b4f77ad8b6ee0e4229596_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190a02ec29f99b4f77ad8b6ee0e4229596_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:27,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/b015e7a61604443192f4e4f0aac3ee56, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:27,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/b015e7a61604443192f4e4f0aac3ee56 is 175, key is test_row_0/A:col10/1732029267094/Put/seqid=0 2024-11-19T15:14:27,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742033_1209 (size=31255) 2024-11-19T15:14:27,832 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/b015e7a61604443192f4e4f0aac3ee56 2024-11-19T15:14:27,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/38bde917b22f4a10a3fee81c73426cfc is 50, key is test_row_0/B:col10/1732029267094/Put/seqid=0 2024-11-19T15:14:27,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742034_1210 (size=12301) 2024-11-19T15:14:27,870 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/38bde917b22f4a10a3fee81c73426cfc 2024-11-19T15:14:27,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/6b1620acf1b44c26a47aa9784128e132 is 50, key is test_row_0/C:col10/1732029267094/Put/seqid=0 2024-11-19T15:14:27,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742035_1211 (size=12301) 2024-11-19T15:14:27,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029327914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029327915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029327915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T15:14:28,117 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#A#compaction#172 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:28,118 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/cd832f9bab6a47edb105584e85491370 is 175, key is test_row_0/A:col10/1732029267067/Put/seqid=0 2024-11-19T15:14:28,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029328125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:28,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029328125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029328125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742036_1212 (size=31869) 2024-11-19T15:14:28,145 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/cd832f9bab6a47edb105584e85491370 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/cd832f9bab6a47edb105584e85491370 2024-11-19T15:14:28,150 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/A of f012bfcce8bd6aef12a49ca6ef302f96 into cd832f9bab6a47edb105584e85491370(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:28,150 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:28,150 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/A, priority=11, startTime=1732029267635; duration=0sec 2024-11-19T15:14:28,150 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:28,150 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:A 2024-11-19T15:14:28,302 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/6b1620acf1b44c26a47aa9784128e132 2024-11-19T15:14:28,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/b015e7a61604443192f4e4f0aac3ee56 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/b015e7a61604443192f4e4f0aac3ee56 2024-11-19T15:14:28,313 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/b015e7a61604443192f4e4f0aac3ee56, entries=150, sequenceid=296, filesize=30.5 K 2024-11-19T15:14:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/38bde917b22f4a10a3fee81c73426cfc as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/38bde917b22f4a10a3fee81c73426cfc 2024-11-19T15:14:28,318 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/38bde917b22f4a10a3fee81c73426cfc, entries=150, sequenceid=296, filesize=12.0 K 2024-11-19T15:14:28,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/6b1620acf1b44c26a47aa9784128e132 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/6b1620acf1b44c26a47aa9784128e132 2024-11-19T15:14:28,323 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/6b1620acf1b44c26a47aa9784128e132, entries=150, sequenceid=296, filesize=12.0 K 2024-11-19T15:14:28,325 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f012bfcce8bd6aef12a49ca6ef302f96 in 582ms, sequenceid=296, compaction requested=false 2024-11-19T15:14:28,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:28,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:28,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-19T15:14:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-19T15:14:28,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T15:14:28,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 893 msec 2024-11-19T15:14:28,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 908 msec 2024-11-19T15:14:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:28,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T15:14:28,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:28,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:28,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:28,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:28,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:28,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:28,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119065f2fee01e54f748b96ff4996da322e_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029268439/Put/seqid=0 2024-11-19T15:14:28,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742037_1213 (size=17534) 2024-11-19T15:14:28,465 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:28,469 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119065f2fee01e54f748b96ff4996da322e_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119065f2fee01e54f748b96ff4996da322e_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:28,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/21f0d3aef10f4305b675e645e2130d7f, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:28,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/21f0d3aef10f4305b675e645e2130d7f is 175, key is test_row_0/A:col10/1732029268439/Put/seqid=0 2024-11-19T15:14:28,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029328477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029328477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029328477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:28,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742038_1214 (size=48639) 2024-11-19T15:14:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T15:14:28,564 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:28,564 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-19T15:14:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T15:14:28,566 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:28,567 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:28,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:28,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029328584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:28,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029328594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:28,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029328594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T15:14:28,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T15:14:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:28,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:28,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029328794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029328804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:28,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029328804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:28,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T15:14:28,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:28,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:28,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:28,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:28,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T15:14:28,885 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/21f0d3aef10f4305b675e645e2130d7f 2024-11-19T15:14:28,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/565f086c7a444fbcb1755a3c980ef80e is 50, key is test_row_0/B:col10/1732029268439/Put/seqid=0 2024-11-19T15:14:28,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742039_1215 (size=12301) 2024-11-19T15:14:29,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T15:14:29,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:29,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029329104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:29,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029329125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:29,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:29,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:29,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029329125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:29,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T15:14:29,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:29,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T15:14:29,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/565f086c7a444fbcb1755a3c980ef80e 2024-11-19T15:14:29,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/7bf52d5ea7bf4712a7b412d1a94d9d0e is 50, key is test_row_0/C:col10/1732029268439/Put/seqid=0 2024-11-19T15:14:29,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742040_1216 (size=12301) 2024-11-19T15:14:29,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T15:14:29,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:29,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T15:14:29,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:29,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029329614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:29,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:29,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:29,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:29,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029329634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:29,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029329634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:29,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:29,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T15:14:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T15:14:29,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/7bf52d5ea7bf4712a7b412d1a94d9d0e 2024-11-19T15:14:29,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/21f0d3aef10f4305b675e645e2130d7f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/21f0d3aef10f4305b675e645e2130d7f 2024-11-19T15:14:29,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/21f0d3aef10f4305b675e645e2130d7f, entries=250, sequenceid=316, filesize=47.5 K 2024-11-19T15:14:29,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/565f086c7a444fbcb1755a3c980ef80e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/565f086c7a444fbcb1755a3c980ef80e 2024-11-19T15:14:29,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/565f086c7a444fbcb1755a3c980ef80e, entries=150, sequenceid=316, filesize=12.0 K 2024-11-19T15:14:29,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/7bf52d5ea7bf4712a7b412d1a94d9d0e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/7bf52d5ea7bf4712a7b412d1a94d9d0e 2024-11-19T15:14:29,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/7bf52d5ea7bf4712a7b412d1a94d9d0e, entries=150, sequenceid=316, filesize=12.0 K 2024-11-19T15:14:29,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for f012bfcce8bd6aef12a49ca6ef302f96 in 1303ms, sequenceid=316, compaction requested=true 2024-11-19T15:14:29,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:29,743 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:29,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:29,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:29,744 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:29,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:29,745 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111763 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:29,745 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/A is initiating minor compaction (all files) 2024-11-19T15:14:29,745 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/A in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,745 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/cd832f9bab6a47edb105584e85491370, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/b015e7a61604443192f4e4f0aac3ee56, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/21f0d3aef10f4305b675e645e2130d7f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=109.1 K 2024-11-19T15:14:29,745 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,745 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/cd832f9bab6a47edb105584e85491370, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/b015e7a61604443192f4e4f0aac3ee56, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/21f0d3aef10f4305b675e645e2130d7f] 2024-11-19T15:14:29,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:29,746 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd832f9bab6a47edb105584e85491370, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732029266723 2024-11-19T15:14:29,746 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:29,746 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/B is initiating minor compaction (all files) 2024-11-19T15:14:29,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:29,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:29,746 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/B in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,746 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/2103259ef9e74851984da34661b9a4d5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/38bde917b22f4a10a3fee81c73426cfc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/565f086c7a444fbcb1755a3c980ef80e] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=36.6 K 2024-11-19T15:14:29,747 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting b015e7a61604443192f4e4f0aac3ee56, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732029267094 2024-11-19T15:14:29,747 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 2103259ef9e74851984da34661b9a4d5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732029266723 2024-11-19T15:14:29,747 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21f0d3aef10f4305b675e645e2130d7f, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732029267765 2024-11-19T15:14:29,747 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 38bde917b22f4a10a3fee81c73426cfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732029267094 2024-11-19T15:14:29,748 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 565f086c7a444fbcb1755a3c980ef80e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732029267765 2024-11-19T15:14:29,756 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:29,756 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#B#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:29,756 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/c87a585586204384a7ced751f6c32acd is 50, key is test_row_0/B:col10/1732029268439/Put/seqid=0 2024-11-19T15:14:29,758 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119a3dcf7f8cacd45089de3477d6fac973d_f012bfcce8bd6aef12a49ca6ef302f96 store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:29,760 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119a3dcf7f8cacd45089de3477d6fac973d_f012bfcce8bd6aef12a49ca6ef302f96, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:29,760 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a3dcf7f8cacd45089de3477d6fac973d_f012bfcce8bd6aef12a49ca6ef302f96 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:29,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742041_1217 (size=13017) 2024-11-19T15:14:29,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742042_1218 (size=4469) 2024-11-19T15:14:29,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T15:14:29,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:29,801 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T15:14:29,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:29,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:29,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:29,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:29,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:29,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:29,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c297d2a46ea74d2a8058f62de9b5bd42_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029268473/Put/seqid=0 2024-11-19T15:14:29,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742043_1219 (size=12454) 2024-11-19T15:14:30,176 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/c87a585586204384a7ced751f6c32acd as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c87a585586204384a7ced751f6c32acd 2024-11-19T15:14:30,176 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#A#compaction#181 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:30,177 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/eda4b29e73c74dfc96babc6acd665339 is 175, key is test_row_0/A:col10/1732029268439/Put/seqid=0 2024-11-19T15:14:30,183 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/B of f012bfcce8bd6aef12a49ca6ef302f96 into c87a585586204384a7ced751f6c32acd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:30,183 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:30,183 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/B, priority=13, startTime=1732029269744; duration=0sec 2024-11-19T15:14:30,184 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:30,184 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:B 2024-11-19T15:14:30,184 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:30,185 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:30,185 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/C is initiating minor compaction (all files) 2024-11-19T15:14:30,185 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/C in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:30,185 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/fe0b21c4529540a8b62991155da87970, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/6b1620acf1b44c26a47aa9784128e132, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/7bf52d5ea7bf4712a7b412d1a94d9d0e] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=36.6 K 2024-11-19T15:14:30,185 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting fe0b21c4529540a8b62991155da87970, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732029266723 2024-11-19T15:14:30,186 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b1620acf1b44c26a47aa9784128e132, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732029267094 2024-11-19T15:14:30,187 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf52d5ea7bf4712a7b412d1a94d9d0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732029267765 2024-11-19T15:14:30,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742044_1220 (size=31971) 2024-11-19T15:14:30,201 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/eda4b29e73c74dfc96babc6acd665339 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/eda4b29e73c74dfc96babc6acd665339 2024-11-19T15:14:30,210 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/A of f012bfcce8bd6aef12a49ca6ef302f96 into eda4b29e73c74dfc96babc6acd665339(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:30,210 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:30,210 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/A, priority=13, startTime=1732029269743; duration=0sec 2024-11-19T15:14:30,210 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:30,210 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:A 2024-11-19T15:14:30,212 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:30,213 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/cf9b494a4adb4fdeaae5e160f383c748 is 50, key is test_row_0/C:col10/1732029268439/Put/seqid=0 2024-11-19T15:14:30,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742045_1221 (size=13017) 2024-11-19T15:14:30,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:30,228 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/cf9b494a4adb4fdeaae5e160f383c748 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/cf9b494a4adb4fdeaae5e160f383c748 2024-11-19T15:14:30,229 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c297d2a46ea74d2a8058f62de9b5bd42_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c297d2a46ea74d2a8058f62de9b5bd42_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:30,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/ffea7591f8694619a465d26bfb7c1bd2, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:30,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/ffea7591f8694619a465d26bfb7c1bd2 is 175, key is test_row_0/A:col10/1732029268473/Put/seqid=0 2024-11-19T15:14:30,238 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/C of f012bfcce8bd6aef12a49ca6ef302f96 into cf9b494a4adb4fdeaae5e160f383c748(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:30,238 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:30,238 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/C, priority=13, startTime=1732029269746; duration=0sec 2024-11-19T15:14:30,238 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:30,238 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:C 2024-11-19T15:14:30,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742046_1222 (size=31255) 2024-11-19T15:14:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:30,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. as already flushing 2024-11-19T15:14:30,641 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/ffea7591f8694619a465d26bfb7c1bd2 2024-11-19T15:14:30,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029330654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:30,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:30,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029330658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:30,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/35b2d3f907954127b9f2cf0e9ac7d402 is 50, key is test_row_0/B:col10/1732029268473/Put/seqid=0 2024-11-19T15:14:30,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029330660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:30,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742047_1223 (size=12301) 2024-11-19T15:14:30,676 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/35b2d3f907954127b9f2cf0e9ac7d402 2024-11-19T15:14:30,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/f2c41d3396cf40f296d1ee7c31c4883e is 50, key is test_row_0/C:col10/1732029268473/Put/seqid=0 2024-11-19T15:14:30,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742048_1224 (size=12301) 2024-11-19T15:14:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T15:14:30,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029330764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:30,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:30,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029330764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:30,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:30,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029330775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:30,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029330974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:30,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029330974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:30,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:30,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029330984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:31,092 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/f2c41d3396cf40f296d1ee7c31c4883e 2024-11-19T15:14:31,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/ffea7591f8694619a465d26bfb7c1bd2 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ffea7591f8694619a465d26bfb7c1bd2 2024-11-19T15:14:31,108 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ffea7591f8694619a465d26bfb7c1bd2, entries=150, sequenceid=332, filesize=30.5 K 2024-11-19T15:14:31,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/35b2d3f907954127b9f2cf0e9ac7d402 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/35b2d3f907954127b9f2cf0e9ac7d402 2024-11-19T15:14:31,113 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/35b2d3f907954127b9f2cf0e9ac7d402, entries=150, sequenceid=332, filesize=12.0 K 2024-11-19T15:14:31,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/f2c41d3396cf40f296d1ee7c31c4883e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/f2c41d3396cf40f296d1ee7c31c4883e 2024-11-19T15:14:31,119 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/f2c41d3396cf40f296d1ee7c31c4883e, entries=150, sequenceid=332, filesize=12.0 K 2024-11-19T15:14:31,121 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f012bfcce8bd6aef12a49ca6ef302f96 in 1320ms, sequenceid=332, compaction requested=false 2024-11-19T15:14:31,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:31,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:31,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-19T15:14:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-19T15:14:31,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T15:14:31,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5560 sec 2024-11-19T15:14:31,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.5640 sec 2024-11-19T15:14:31,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T15:14:31,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:31,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:31,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:31,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:31,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:31,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:31,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196e2474c179354dd489488496aa3ffbe6_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029271286/Put/seqid=0 2024-11-19T15:14:31,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029331318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:31,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029331319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:31,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029331319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:31,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742049_1225 (size=14994) 2024-11-19T15:14:31,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029331435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:31,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029331435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029331435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:31,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029331644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:31,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029331644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029331645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:31,758 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:31,762 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196e2474c179354dd489488496aa3ffbe6_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196e2474c179354dd489488496aa3ffbe6_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:31,763 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/17ba266c877e43ad98435f35d65673bb, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:31,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/17ba266c877e43ad98435f35d65673bb is 175, key is test_row_0/A:col10/1732029271286/Put/seqid=0 2024-11-19T15:14:31,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742050_1226 (size=39949) 2024-11-19T15:14:31,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029331954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:31,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029331964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029331964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:31,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:32,168 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/17ba266c877e43ad98435f35d65673bb 2024-11-19T15:14:32,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53196 deadline: 1732029332174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:32,176 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in f012bfcce8bd6aef12a49ca6ef302f96 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:32,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/641f6fd78f034d14aed3e058ddc4a2eb is 50, key is test_row_0/B:col10/1732029271286/Put/seqid=0 2024-11-19T15:14:32,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742051_1227 (size=12301) 2024-11-19T15:14:32,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/641f6fd78f034d14aed3e058ddc4a2eb 2024-11-19T15:14:32,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:32,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/31dc6e8db010480a8c4ee4903bea5505 is 50, key is test_row_0/C:col10/1732029271286/Put/seqid=0 2024-11-19T15:14:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53150 deadline: 1732029332194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:32,195 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in f012bfcce8bd6aef12a49ca6ef302f96 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:32,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742052_1228 (size=12301) 2024-11-19T15:14:32,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53138 deadline: 1732029332464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:32,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732029332474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:32,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:32,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:32,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53192 deadline: 1732029332474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:32,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:14:32,500 INFO [Thread-670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-670. 2024-11-19T15:14:32,500 DEBUG [Thread-670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:32,500 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:32,501 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:32,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:32,502 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-11-19T15:14:32,502 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:32,502 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:32,502 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:32,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:32,506 INFO [Thread-668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-668. 2024-11-19T15:14:32,506 DEBUG [Thread-668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:32,506 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:32,506 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:32,506 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:32,509 INFO [Thread-674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-674. 2024-11-19T15:14:32,509 DEBUG [Thread-674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:32,509 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:32,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:32,509 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:32,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/31dc6e8db010480a8c4ee4903bea5505 2024-11-19T15:14:32,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/17ba266c877e43ad98435f35d65673bb as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/17ba266c877e43ad98435f35d65673bb 2024-11-19T15:14:32,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/17ba266c877e43ad98435f35d65673bb, entries=200, sequenceid=352, filesize=39.0 K 2024-11-19T15:14:32,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/641f6fd78f034d14aed3e058ddc4a2eb as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/641f6fd78f034d14aed3e058ddc4a2eb 2024-11-19T15:14:32,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/641f6fd78f034d14aed3e058ddc4a2eb, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T15:14:32,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/31dc6e8db010480a8c4ee4903bea5505 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/31dc6e8db010480a8c4ee4903bea5505 2024-11-19T15:14:32,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/31dc6e8db010480a8c4ee4903bea5505, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T15:14:32,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f012bfcce8bd6aef12a49ca6ef302f96 in 1342ms, sequenceid=352, compaction requested=true 2024-11-19T15:14:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:32,630 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f012bfcce8bd6aef12a49ca6ef302f96:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:32,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:32,630 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:32,631 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:32,631 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:32,631 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/B is initiating minor compaction (all files) 2024-11-19T15:14:32,631 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/A is initiating minor compaction (all files) 2024-11-19T15:14:32,631 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/B in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:32,632 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/A in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:32,632 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c87a585586204384a7ced751f6c32acd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/35b2d3f907954127b9f2cf0e9ac7d402, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/641f6fd78f034d14aed3e058ddc4a2eb] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=36.7 K 2024-11-19T15:14:32,632 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/eda4b29e73c74dfc96babc6acd665339, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ffea7591f8694619a465d26bfb7c1bd2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/17ba266c877e43ad98435f35d65673bb] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=100.8 K 2024-11-19T15:14:32,632 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:32,632 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/eda4b29e73c74dfc96babc6acd665339, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ffea7591f8694619a465d26bfb7c1bd2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/17ba266c877e43ad98435f35d65673bb] 2024-11-19T15:14:32,632 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting c87a585586204384a7ced751f6c32acd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732029267765 2024-11-19T15:14:32,632 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting eda4b29e73c74dfc96babc6acd665339, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732029267765 2024-11-19T15:14:32,632 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 35b2d3f907954127b9f2cf0e9ac7d402, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732029268454 2024-11-19T15:14:32,633 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 641f6fd78f034d14aed3e058ddc4a2eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732029270650 2024-11-19T15:14:32,633 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffea7591f8694619a465d26bfb7c1bd2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732029268454 2024-11-19T15:14:32,633 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17ba266c877e43ad98435f35d65673bb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732029270650 2024-11-19T15:14:32,650 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#B#compaction#189 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:32,651 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/d0429bc4db9e448fbe20f657e7b25114 is 50, key is test_row_0/B:col10/1732029271286/Put/seqid=0 2024-11-19T15:14:32,651 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:32,654 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411194d44793523364dbaa4cef3fb832b177d_f012bfcce8bd6aef12a49ca6ef302f96 store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:32,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742053_1229 (size=13119) 2024-11-19T15:14:32,684 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411194d44793523364dbaa4cef3fb832b177d_f012bfcce8bd6aef12a49ca6ef302f96, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:32,684 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194d44793523364dbaa4cef3fb832b177d_f012bfcce8bd6aef12a49ca6ef302f96 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:32,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742054_1230 (size=4469) 2024-11-19T15:14:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T15:14:32,725 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:33,071 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/d0429bc4db9e448fbe20f657e7b25114 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/d0429bc4db9e448fbe20f657e7b25114 2024-11-19T15:14:33,077 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/B of f012bfcce8bd6aef12a49ca6ef302f96 into d0429bc4db9e448fbe20f657e7b25114(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:33,077 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:33,077 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/B, priority=13, startTime=1732029272630; duration=0sec 2024-11-19T15:14:33,077 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:33,077 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:B 2024-11-19T15:14:33,077 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:33,078 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:33,078 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): f012bfcce8bd6aef12a49ca6ef302f96/C is initiating minor compaction (all files) 2024-11-19T15:14:33,079 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f012bfcce8bd6aef12a49ca6ef302f96/C in TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:33,079 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/cf9b494a4adb4fdeaae5e160f383c748, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/f2c41d3396cf40f296d1ee7c31c4883e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/31dc6e8db010480a8c4ee4903bea5505] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp, totalSize=36.7 K 2024-11-19T15:14:33,079 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting cf9b494a4adb4fdeaae5e160f383c748, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732029267765 2024-11-19T15:14:33,080 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c41d3396cf40f296d1ee7c31c4883e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732029268454 2024-11-19T15:14:33,080 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 31dc6e8db010480a8c4ee4903bea5505, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732029270650 2024-11-19T15:14:33,090 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#A#compaction#190 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:33,091 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/c92ef999608f403a84a5f937cf60b7d4 is 175, key is test_row_0/A:col10/1732029271286/Put/seqid=0 2024-11-19T15:14:33,093 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f012bfcce8bd6aef12a49ca6ef302f96#C#compaction#191 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:33,093 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/ded8c3d1dbf3448c988c05063a09d981 is 50, key is test_row_0/C:col10/1732029271286/Put/seqid=0 2024-11-19T15:14:33,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742055_1231 (size=32073) 2024-11-19T15:14:33,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742056_1232 (size=13119) 2024-11-19T15:14:33,124 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/ded8c3d1dbf3448c988c05063a09d981 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/ded8c3d1dbf3448c988c05063a09d981 2024-11-19T15:14:33,130 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/C of f012bfcce8bd6aef12a49ca6ef302f96 into ded8c3d1dbf3448c988c05063a09d981(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:33,130 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:33,130 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/C, priority=13, startTime=1732029272630; duration=0sec 2024-11-19T15:14:33,130 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:33,130 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:C 2024-11-19T15:14:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:33,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T15:14:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:33,479 INFO [Thread-665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-665. 2024-11-19T15:14:33,479 DEBUG [Thread-665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:33,479 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:33,479 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:33,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:33,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bc80fdb2032a457dbf0e7d559d63d3f9_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_0/A:col10/1732029271315/Put/seqid=0 2024-11-19T15:14:33,496 INFO [Thread-657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-657. 2024-11-19T15:14:33,496 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-11-19T15:14:33,496 DEBUG [Thread-657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:33,496 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:33,496 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:33,496 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:33,496 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:33,496 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:33,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:33,497 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:33,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742057_1233 (size=12454) 2024-11-19T15:14:33,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:33,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bc80fdb2032a457dbf0e7d559d63d3f9_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bc80fdb2032a457dbf0e7d559d63d3f9_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:33,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/a208f89f95bf4a319f5bf5d118e9f26b, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:33,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/a208f89f95bf4a319f5bf5d118e9f26b is 175, key is test_row_0/A:col10/1732029271315/Put/seqid=0 2024-11-19T15:14:33,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742058_1234 (size=31255) 2024-11-19T15:14:33,523 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/c92ef999608f403a84a5f937cf60b7d4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c92ef999608f403a84a5f937cf60b7d4 2024-11-19T15:14:33,530 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f012bfcce8bd6aef12a49ca6ef302f96/A of f012bfcce8bd6aef12a49ca6ef302f96 into c92ef999608f403a84a5f937cf60b7d4(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:33,530 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:33,530 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., storeName=f012bfcce8bd6aef12a49ca6ef302f96/A, priority=13, startTime=1732029272630; duration=0sec 2024-11-19T15:14:33,530 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:33,531 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f012bfcce8bd6aef12a49ca6ef302f96:A 2024-11-19T15:14:33,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/a208f89f95bf4a319f5bf5d118e9f26b 2024-11-19T15:14:33,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/b04e1a4eabd841789615dae51ea8adf6 is 50, key is test_row_0/B:col10/1732029271315/Put/seqid=0 2024-11-19T15:14:33,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742059_1235 (size=12301) 2024-11-19T15:14:34,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/b04e1a4eabd841789615dae51ea8adf6 2024-11-19T15:14:34,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/61c5cfc086684567b004bb358f01e443 is 50, key is test_row_0/C:col10/1732029271315/Put/seqid=0 2024-11-19T15:14:34,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742060_1236 (size=12301) 2024-11-19T15:14:34,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/61c5cfc086684567b004bb358f01e443 2024-11-19T15:14:34,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/a208f89f95bf4a319f5bf5d118e9f26b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/a208f89f95bf4a319f5bf5d118e9f26b 2024-11-19T15:14:34,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/a208f89f95bf4a319f5bf5d118e9f26b, entries=150, sequenceid=370, filesize=30.5 K 2024-11-19T15:14:34,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/b04e1a4eabd841789615dae51ea8adf6 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/b04e1a4eabd841789615dae51ea8adf6 2024-11-19T15:14:34,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/b04e1a4eabd841789615dae51ea8adf6, entries=150, sequenceid=370, filesize=12.0 K 2024-11-19T15:14:34,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/61c5cfc086684567b004bb358f01e443 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/61c5cfc086684567b004bb358f01e443 2024-11-19T15:14:34,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/61c5cfc086684567b004bb358f01e443, entries=150, sequenceid=370, filesize=12.0 K 2024-11-19T15:14:34,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=13.42 KB/13740 for f012bfcce8bd6aef12a49ca6ef302f96 in 1297ms, sequenceid=370, compaction requested=false 2024-11-19T15:14:34,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f012bfcce8bd6aef12a49ca6ef302f96: 2024-11-19T15:14:36,979 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T15:14:42,217 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-11-19T15:14:42,218 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:42,218 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:42,218 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:42,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:42,286 INFO [Thread-663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-663. 2024-11-19T15:14:42,286 DEBUG [Thread-663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:14:42,286 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:42,286 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:42,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2810 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2730 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1212 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3635 rows 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1211 2024-11-19T15:14:42,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3630 rows 2024-11-19T15:14:42,287 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T15:14:42,287 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T15:14:42,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:42,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:42,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:14:42,288 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T15:14:42,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T15:14:42,291 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029282291"}]},"ts":"1732029282291"} 2024-11-19T15:14:42,292 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T15:14:42,292 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T15:14:42,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T15:14:42,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, UNASSIGN}] 2024-11-19T15:14:42,295 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, UNASSIGN 2024-11-19T15:14:42,296 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=f012bfcce8bd6aef12a49ca6ef302f96, regionState=CLOSING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:42,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, UNASSIGN because future has completed 2024-11-19T15:14:42,297 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T15:14:42,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:14:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T15:14:42,450 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:42,450 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T15:14:42,450 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing f012bfcce8bd6aef12a49ca6ef302f96, disabling compactions & flushes 2024-11-19T15:14:42,450 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:42,450 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:42,450 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. after waiting 0 ms 2024-11-19T15:14:42,450 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:42,450 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing f012bfcce8bd6aef12a49ca6ef302f96 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T15:14:42,450 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=A 2024-11-19T15:14:42,451 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:42,451 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=B 2024-11-19T15:14:42,451 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:42,451 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f012bfcce8bd6aef12a49ca6ef302f96, store=C 2024-11-19T15:14:42,451 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:42,457 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199b89e6e1e961408e910cc1517cc05ca5_f012bfcce8bd6aef12a49ca6ef302f96 is 50, key is test_row_1/A:col10/1732029282215/Put/seqid=0 2024-11-19T15:14:42,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742061_1237 (size=9914) 2024-11-19T15:14:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T15:14:42,863 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:42,868 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199b89e6e1e961408e910cc1517cc05ca5_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b89e6e1e961408e910cc1517cc05ca5_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:42,869 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/1fa7ca71b62f4043893f2c412f16e043, store: [table=TestAcidGuarantees family=A region=f012bfcce8bd6aef12a49ca6ef302f96] 2024-11-19T15:14:42,869 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/1fa7ca71b62f4043893f2c412f16e043 is 175, key is test_row_1/A:col10/1732029282215/Put/seqid=0 2024-11-19T15:14:42,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742062_1238 (size=22561) 2024-11-19T15:14:42,877 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/1fa7ca71b62f4043893f2c412f16e043 2024-11-19T15:14:42,885 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/a5214cdd1503429380fff5495b4a0436 is 50, key is test_row_1/B:col10/1732029282215/Put/seqid=0 2024-11-19T15:14:42,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742063_1239 (size=9857) 2024-11-19T15:14:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T15:14:43,290 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/a5214cdd1503429380fff5495b4a0436 2024-11-19T15:14:43,299 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/94b6cc9bd26449d4893d9c3ed2eb63c9 is 50, key is test_row_1/C:col10/1732029282215/Put/seqid=0 2024-11-19T15:14:43,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742064_1240 (size=9857) 2024-11-19T15:14:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T15:14:43,707 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/94b6cc9bd26449d4893d9c3ed2eb63c9 2024-11-19T15:14:43,713 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/A/1fa7ca71b62f4043893f2c412f16e043 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/1fa7ca71b62f4043893f2c412f16e043 2024-11-19T15:14:43,717 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/1fa7ca71b62f4043893f2c412f16e043, entries=100, sequenceid=378, filesize=22.0 K 2024-11-19T15:14:43,718 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/B/a5214cdd1503429380fff5495b4a0436 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/a5214cdd1503429380fff5495b4a0436 2024-11-19T15:14:43,721 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/a5214cdd1503429380fff5495b4a0436, entries=100, sequenceid=378, filesize=9.6 K 2024-11-19T15:14:43,722 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/.tmp/C/94b6cc9bd26449d4893d9c3ed2eb63c9 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/94b6cc9bd26449d4893d9c3ed2eb63c9 2024-11-19T15:14:43,726 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/94b6cc9bd26449d4893d9c3ed2eb63c9, entries=100, sequenceid=378, filesize=9.6 K 2024-11-19T15:14:43,727 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f012bfcce8bd6aef12a49ca6ef302f96 in 1277ms, sequenceid=378, compaction requested=true 2024-11-19T15:14:43,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/d3db53e766ba49af813d962c6f990365, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c1af040d39484063991a77015d20dcb2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/24a698f38c2a4c0dbe035eed759dfdc0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c7569166fd5a4673bedbe6cee0e170a1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2662366243764452b21379ffdfedf2b5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/38cf53432e9345b680416d8679ddf400, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/0d293598b3a547bc96e4f6cb33e01c11, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/1e10c31bfc8943bc98dd892352d5c747, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/4a18c1af4d8f496e89f929c8c4e2a07a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/26187bbd77674b38bfddd048ae64aaa0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e1c94128e0da405fa84fed22c81c0e88, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/96b42e41b71349b697addffb412af8e8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/895010eb07034f2d8c08cb0c833edc87, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/9d2bef2b22ea4b26a57ac9dadd2f93b3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2c18a8365b0848e9906c05b2b942969b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/830e849b742a4a72b7beb911ef26b042, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e807cca4b2994857ba8fc06a08adac9a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ed3b5e5dfa634c59b1665ba9783eaaec, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/cd832f9bab6a47edb105584e85491370, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/b015e7a61604443192f4e4f0aac3ee56, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/21f0d3aef10f4305b675e645e2130d7f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/eda4b29e73c74dfc96babc6acd665339, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ffea7591f8694619a465d26bfb7c1bd2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/17ba266c877e43ad98435f35d65673bb] to archive 2024-11-19T15:14:43,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:14:43,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/d3db53e766ba49af813d962c6f990365 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/d3db53e766ba49af813d962c6f990365 2024-11-19T15:14:43,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c1af040d39484063991a77015d20dcb2 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c1af040d39484063991a77015d20dcb2 2024-11-19T15:14:43,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/24a698f38c2a4c0dbe035eed759dfdc0 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/24a698f38c2a4c0dbe035eed759dfdc0 2024-11-19T15:14:43,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c7569166fd5a4673bedbe6cee0e170a1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c7569166fd5a4673bedbe6cee0e170a1 2024-11-19T15:14:43,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2662366243764452b21379ffdfedf2b5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2662366243764452b21379ffdfedf2b5 2024-11-19T15:14:43,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/38cf53432e9345b680416d8679ddf400 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/38cf53432e9345b680416d8679ddf400 2024-11-19T15:14:43,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/0d293598b3a547bc96e4f6cb33e01c11 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/0d293598b3a547bc96e4f6cb33e01c11 2024-11-19T15:14:43,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/1e10c31bfc8943bc98dd892352d5c747 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/1e10c31bfc8943bc98dd892352d5c747 2024-11-19T15:14:43,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/4a18c1af4d8f496e89f929c8c4e2a07a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/4a18c1af4d8f496e89f929c8c4e2a07a 2024-11-19T15:14:43,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/26187bbd77674b38bfddd048ae64aaa0 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/26187bbd77674b38bfddd048ae64aaa0 2024-11-19T15:14:43,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e1c94128e0da405fa84fed22c81c0e88 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e1c94128e0da405fa84fed22c81c0e88 2024-11-19T15:14:43,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/96b42e41b71349b697addffb412af8e8 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/96b42e41b71349b697addffb412af8e8 2024-11-19T15:14:43,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/895010eb07034f2d8c08cb0c833edc87 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/895010eb07034f2d8c08cb0c833edc87 2024-11-19T15:14:43,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/9d2bef2b22ea4b26a57ac9dadd2f93b3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/9d2bef2b22ea4b26a57ac9dadd2f93b3 2024-11-19T15:14:43,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2c18a8365b0848e9906c05b2b942969b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/2c18a8365b0848e9906c05b2b942969b 2024-11-19T15:14:43,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/830e849b742a4a72b7beb911ef26b042 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/830e849b742a4a72b7beb911ef26b042 2024-11-19T15:14:43,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e807cca4b2994857ba8fc06a08adac9a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/e807cca4b2994857ba8fc06a08adac9a 2024-11-19T15:14:43,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ed3b5e5dfa634c59b1665ba9783eaaec to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ed3b5e5dfa634c59b1665ba9783eaaec 2024-11-19T15:14:43,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/cd832f9bab6a47edb105584e85491370 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/cd832f9bab6a47edb105584e85491370 2024-11-19T15:14:43,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/b015e7a61604443192f4e4f0aac3ee56 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/b015e7a61604443192f4e4f0aac3ee56 2024-11-19T15:14:43,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/21f0d3aef10f4305b675e645e2130d7f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/21f0d3aef10f4305b675e645e2130d7f 2024-11-19T15:14:43,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/eda4b29e73c74dfc96babc6acd665339 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/eda4b29e73c74dfc96babc6acd665339 2024-11-19T15:14:43,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ffea7591f8694619a465d26bfb7c1bd2 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/ffea7591f8694619a465d26bfb7c1bd2 2024-11-19T15:14:43,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/17ba266c877e43ad98435f35d65673bb to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/17ba266c877e43ad98435f35d65673bb 2024-11-19T15:14:43,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/fe968d1ad042478e88e9444071b1672a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/775b1a42bce04c0bb3decc50f46ff359, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/3c5b318be2364ded9d4335d30d15fde4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/fb770ebf0ae0429f918b58e1cdfdd8e7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/d88fc8e0d89948c2802f66fd46883d9c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/4009ffdf99f14c39b1179e5a7f6cc0d5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/f18fb8a7f2044945a0b7b03809518b1f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c4357cb516d04296a9a4e5f62ef18937, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/6ed132b1f12d465dba35238b1dfeb495, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c4185c90751d4f948d8c02e97373ee51, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/512d9a2ec3de4bf68d1d572559f09d8b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/717da90e65b4467b8de627f37bca4faf, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/622822b6f06c4d70af342e19cd1f4557, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/eab65e31bf8444aa94af5d4379b83826, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/3249eeb2d6784009a96a9bae0a14ecbd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/4271f555e0d5445c8499a58a8c535e2c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/720026d741154ed28c51f98e05365350, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/2103259ef9e74851984da34661b9a4d5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/ee1c2596649a4314843858d09248b1b3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/38bde917b22f4a10a3fee81c73426cfc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c87a585586204384a7ced751f6c32acd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/565f086c7a444fbcb1755a3c980ef80e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/35b2d3f907954127b9f2cf0e9ac7d402, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/641f6fd78f034d14aed3e058ddc4a2eb] to archive 2024-11-19T15:14:43,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:14:43,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/fe968d1ad042478e88e9444071b1672a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/fe968d1ad042478e88e9444071b1672a 2024-11-19T15:14:43,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/775b1a42bce04c0bb3decc50f46ff359 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/775b1a42bce04c0bb3decc50f46ff359 2024-11-19T15:14:43,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/3c5b318be2364ded9d4335d30d15fde4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/3c5b318be2364ded9d4335d30d15fde4 2024-11-19T15:14:43,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/fb770ebf0ae0429f918b58e1cdfdd8e7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/fb770ebf0ae0429f918b58e1cdfdd8e7 2024-11-19T15:14:43,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/d88fc8e0d89948c2802f66fd46883d9c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/d88fc8e0d89948c2802f66fd46883d9c 2024-11-19T15:14:43,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/4009ffdf99f14c39b1179e5a7f6cc0d5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/4009ffdf99f14c39b1179e5a7f6cc0d5 2024-11-19T15:14:43,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/f18fb8a7f2044945a0b7b03809518b1f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/f18fb8a7f2044945a0b7b03809518b1f 2024-11-19T15:14:43,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c4357cb516d04296a9a4e5f62ef18937 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c4357cb516d04296a9a4e5f62ef18937 2024-11-19T15:14:43,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/6ed132b1f12d465dba35238b1dfeb495 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/6ed132b1f12d465dba35238b1dfeb495 2024-11-19T15:14:43,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c4185c90751d4f948d8c02e97373ee51 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c4185c90751d4f948d8c02e97373ee51 2024-11-19T15:14:43,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/512d9a2ec3de4bf68d1d572559f09d8b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/512d9a2ec3de4bf68d1d572559f09d8b 2024-11-19T15:14:43,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/717da90e65b4467b8de627f37bca4faf to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/717da90e65b4467b8de627f37bca4faf 2024-11-19T15:14:43,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/622822b6f06c4d70af342e19cd1f4557 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/622822b6f06c4d70af342e19cd1f4557 2024-11-19T15:14:43,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/eab65e31bf8444aa94af5d4379b83826 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/eab65e31bf8444aa94af5d4379b83826 2024-11-19T15:14:43,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/3249eeb2d6784009a96a9bae0a14ecbd to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/3249eeb2d6784009a96a9bae0a14ecbd 2024-11-19T15:14:43,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/4271f555e0d5445c8499a58a8c535e2c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/4271f555e0d5445c8499a58a8c535e2c 2024-11-19T15:14:43,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/720026d741154ed28c51f98e05365350 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/720026d741154ed28c51f98e05365350 2024-11-19T15:14:43,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/2103259ef9e74851984da34661b9a4d5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/2103259ef9e74851984da34661b9a4d5 2024-11-19T15:14:43,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/ee1c2596649a4314843858d09248b1b3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/ee1c2596649a4314843858d09248b1b3 2024-11-19T15:14:43,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/38bde917b22f4a10a3fee81c73426cfc to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/38bde917b22f4a10a3fee81c73426cfc 2024-11-19T15:14:43,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c87a585586204384a7ced751f6c32acd to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/c87a585586204384a7ced751f6c32acd 2024-11-19T15:14:43,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/565f086c7a444fbcb1755a3c980ef80e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/565f086c7a444fbcb1755a3c980ef80e 2024-11-19T15:14:43,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/35b2d3f907954127b9f2cf0e9ac7d402 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/35b2d3f907954127b9f2cf0e9ac7d402 2024-11-19T15:14:43,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/641f6fd78f034d14aed3e058ddc4a2eb to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/641f6fd78f034d14aed3e058ddc4a2eb 2024-11-19T15:14:43,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/c6247f6e18124c71a2e2a83ef2bb25a9, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/4e86e10495e7459a88f68a346502c5db, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/6da625c8ae184472859d96d9de39d321, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/7f7c93095580419f842189d2405d0394, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/f84bc52d160b4e7fa9c57a3c7efe26d7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/68003efe5b2940a5ba7e8859ce892796, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/0869ae7cf21c48c3b7fe6d55924153d4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/41319eeca1804091810d23405c69d01c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/e160f099f6714af1b6319ef5785ebcbb, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/82748c186c404062adbf47f532428554, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/1d0b3c1fd3f2435e91c07b68d1e98618, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/55668fef110844e5932f82483b8b13a1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/2c252e96df514a898a643d9d26629f33, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/56e34391c65c4e4abbabf581dd9c342d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/35ef696c2826446bbac5cd36165cc7c3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/e7a3f140926046bb9294e48b509308a1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/2059e4f7c9a34cf8a75183e156a38e63, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/fe0b21c4529540a8b62991155da87970, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/fca1774c97f649aab3e37792c20cf450, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/6b1620acf1b44c26a47aa9784128e132, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/cf9b494a4adb4fdeaae5e160f383c748, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/7bf52d5ea7bf4712a7b412d1a94d9d0e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/f2c41d3396cf40f296d1ee7c31c4883e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/31dc6e8db010480a8c4ee4903bea5505] to archive 2024-11-19T15:14:43,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:14:43,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/c6247f6e18124c71a2e2a83ef2bb25a9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/c6247f6e18124c71a2e2a83ef2bb25a9 2024-11-19T15:14:43,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/4e86e10495e7459a88f68a346502c5db to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/4e86e10495e7459a88f68a346502c5db 2024-11-19T15:14:43,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/6da625c8ae184472859d96d9de39d321 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/6da625c8ae184472859d96d9de39d321 2024-11-19T15:14:43,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/7f7c93095580419f842189d2405d0394 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/7f7c93095580419f842189d2405d0394 2024-11-19T15:14:43,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/f84bc52d160b4e7fa9c57a3c7efe26d7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/f84bc52d160b4e7fa9c57a3c7efe26d7 2024-11-19T15:14:43,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/68003efe5b2940a5ba7e8859ce892796 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/68003efe5b2940a5ba7e8859ce892796 2024-11-19T15:14:43,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/0869ae7cf21c48c3b7fe6d55924153d4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/0869ae7cf21c48c3b7fe6d55924153d4 2024-11-19T15:14:43,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/41319eeca1804091810d23405c69d01c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/41319eeca1804091810d23405c69d01c 2024-11-19T15:14:43,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/e160f099f6714af1b6319ef5785ebcbb to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/e160f099f6714af1b6319ef5785ebcbb 2024-11-19T15:14:43,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/82748c186c404062adbf47f532428554 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/82748c186c404062adbf47f532428554 2024-11-19T15:14:43,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/1d0b3c1fd3f2435e91c07b68d1e98618 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/1d0b3c1fd3f2435e91c07b68d1e98618 2024-11-19T15:14:43,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/55668fef110844e5932f82483b8b13a1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/55668fef110844e5932f82483b8b13a1 2024-11-19T15:14:43,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/2c252e96df514a898a643d9d26629f33 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/2c252e96df514a898a643d9d26629f33 2024-11-19T15:14:43,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/56e34391c65c4e4abbabf581dd9c342d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/56e34391c65c4e4abbabf581dd9c342d 2024-11-19T15:14:43,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/35ef696c2826446bbac5cd36165cc7c3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/35ef696c2826446bbac5cd36165cc7c3 2024-11-19T15:14:43,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/e7a3f140926046bb9294e48b509308a1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/e7a3f140926046bb9294e48b509308a1 2024-11-19T15:14:43,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/2059e4f7c9a34cf8a75183e156a38e63 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/2059e4f7c9a34cf8a75183e156a38e63 2024-11-19T15:14:43,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/fe0b21c4529540a8b62991155da87970 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/fe0b21c4529540a8b62991155da87970 2024-11-19T15:14:43,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/fca1774c97f649aab3e37792c20cf450 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/fca1774c97f649aab3e37792c20cf450 2024-11-19T15:14:43,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/6b1620acf1b44c26a47aa9784128e132 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/6b1620acf1b44c26a47aa9784128e132 2024-11-19T15:14:43,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/cf9b494a4adb4fdeaae5e160f383c748 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/cf9b494a4adb4fdeaae5e160f383c748 2024-11-19T15:14:43,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/7bf52d5ea7bf4712a7b412d1a94d9d0e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/7bf52d5ea7bf4712a7b412d1a94d9d0e 2024-11-19T15:14:43,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/f2c41d3396cf40f296d1ee7c31c4883e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/f2c41d3396cf40f296d1ee7c31c4883e 2024-11-19T15:14:43,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/31dc6e8db010480a8c4ee4903bea5505 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/31dc6e8db010480a8c4ee4903bea5505 2024-11-19T15:14:43,863 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/recovered.edits/381.seqid, newMaxSeqId=381, maxSeqId=4 2024-11-19T15:14:43,864 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. 2024-11-19T15:14:43,864 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for f012bfcce8bd6aef12a49ca6ef302f96: Waiting for close lock at 1732029282450Running coprocessor pre-close hooks at 1732029282450Disabling compacts and flushes for region at 1732029282450Disabling writes for close at 1732029282450Obtaining lock to block concurrent updates at 1732029282450Preparing flush snapshotting stores in f012bfcce8bd6aef12a49ca6ef302f96 at 1732029282450Finished memstore snapshotting TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732029282451 (+1 ms)Flushing stores of TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96. at 1732029282451Flushing f012bfcce8bd6aef12a49ca6ef302f96/A: creating writer at 1732029282451Flushing mob file f012bfcce8bd6aef12a49ca6ef302f96/A: appending metadata at 1732029282457 (+6 ms)Flushing mob file f012bfcce8bd6aef12a49ca6ef302f96/A: closing flushed file at 1732029282457Flushing f012bfcce8bd6aef12a49ca6ef302f96/A: appending metadata at 1732029282869 (+412 ms)Flushing f012bfcce8bd6aef12a49ca6ef302f96/A: closing flushed file at 1732029282869Flushing f012bfcce8bd6aef12a49ca6ef302f96/B: creating writer at 1732029282881 (+12 ms)Flushing f012bfcce8bd6aef12a49ca6ef302f96/B: appending metadata at 1732029282884 (+3 ms)Flushing f012bfcce8bd6aef12a49ca6ef302f96/B: closing flushed file at 1732029282885 (+1 ms)Flushing f012bfcce8bd6aef12a49ca6ef302f96/C: creating writer at 1732029283295 (+410 ms)Flushing f012bfcce8bd6aef12a49ca6ef302f96/C: appending metadata at 1732029283298 (+3 ms)Flushing f012bfcce8bd6aef12a49ca6ef302f96/C: closing flushed file at 1732029283298Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@486fc44a: reopening flushed file at 1732029283712 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a571fec: reopening flushed file at 1732029283717 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1aea8dd4: reopening flushed file at 1732029283722 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f012bfcce8bd6aef12a49ca6ef302f96 in 1277ms, sequenceid=378, compaction requested=true at 1732029283727 (+5 ms)Writing region close event to WAL at 1732029283860 (+133 ms)Running coprocessor post-close hooks at 1732029283864 (+4 ms)Closed at 1732029283864 2024-11-19T15:14:43,868 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=f012bfcce8bd6aef12a49ca6ef302f96, regionState=CLOSED 2024-11-19T15:14:43,868 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:43,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:14:43,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-19T15:14:43,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure f012bfcce8bd6aef12a49ca6ef302f96, server=50570b5b74e7,46557,1732029218639 in 1.5730 sec 2024-11-19T15:14:43,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-19T15:14:43,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f012bfcce8bd6aef12a49ca6ef302f96, UNASSIGN in 1.5780 sec 2024-11-19T15:14:43,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-19T15:14:43,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5820 sec 2024-11-19T15:14:43,879 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029283878"}]},"ts":"1732029283878"} 2024-11-19T15:14:43,880 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T15:14:43,880 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T15:14:43,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5930 sec 2024-11-19T15:14:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T15:14:44,435 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:44,435 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T15:14:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:44,437 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T15:14:44,437 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:44,440 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,442 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/recovered.edits] 2024-11-19T15:14:44,444 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/1fa7ca71b62f4043893f2c412f16e043 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/1fa7ca71b62f4043893f2c412f16e043 2024-11-19T15:14:44,445 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/a208f89f95bf4a319f5bf5d118e9f26b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/a208f89f95bf4a319f5bf5d118e9f26b 2024-11-19T15:14:44,447 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c92ef999608f403a84a5f937cf60b7d4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/A/c92ef999608f403a84a5f937cf60b7d4 2024-11-19T15:14:44,449 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/a5214cdd1503429380fff5495b4a0436 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/a5214cdd1503429380fff5495b4a0436 2024-11-19T15:14:44,450 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/b04e1a4eabd841789615dae51ea8adf6 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/b04e1a4eabd841789615dae51ea8adf6 2024-11-19T15:14:44,451 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/d0429bc4db9e448fbe20f657e7b25114 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/B/d0429bc4db9e448fbe20f657e7b25114 2024-11-19T15:14:44,453 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/61c5cfc086684567b004bb358f01e443 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/61c5cfc086684567b004bb358f01e443 2024-11-19T15:14:44,454 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/94b6cc9bd26449d4893d9c3ed2eb63c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/94b6cc9bd26449d4893d9c3ed2eb63c9 2024-11-19T15:14:44,456 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/ded8c3d1dbf3448c988c05063a09d981 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/C/ded8c3d1dbf3448c988c05063a09d981 2024-11-19T15:14:44,458 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/recovered.edits/381.seqid to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96/recovered.edits/381.seqid 2024-11-19T15:14:44,459 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,459 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T15:14:44,459 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T15:14:44,460 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T15:14:44,463 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119065f2fee01e54f748b96ff4996da322e_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119065f2fee01e54f748b96ff4996da322e_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,464 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190a02ec29f99b4f77ad8b6ee0e4229596_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190a02ec29f99b4f77ad8b6ee0e4229596_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,466 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195134c3a51d774f12850f815f06b43c25_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195134c3a51d774f12850f815f06b43c25_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,467 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195db420ce8bfc44218685516429df5672_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195db420ce8bfc44218685516429df5672_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,468 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196c72976759de45718250c9c63c81e08a_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196c72976759de45718250c9c63c81e08a_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,469 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196e2474c179354dd489488496aa3ffbe6_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196e2474c179354dd489488496aa3ffbe6_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,470 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197f88bfa2360b418e8d3570d32583aba6_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197f88bfa2360b418e8d3570d32583aba6_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,471 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111989dcc44bf88d44a2a135246009661bf4_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111989dcc44bf88d44a2a135246009661bf4_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,472 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198fefa0c3e8f04af5b192885b94e35259_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198fefa0c3e8f04af5b192885b94e35259_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,473 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b89e6e1e961408e910cc1517cc05ca5_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b89e6e1e961408e910cc1517cc05ca5_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,474 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a739134e44f14ae1a5a9164258c8b297_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a739134e44f14ae1a5a9164258c8b297_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,475 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a75ec5c20f934656ba5887faf5aa2a3e_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a75ec5c20f934656ba5887faf5aa2a3e_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,476 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b80e908694c44c5490a9ea9a7653f0e7_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b80e908694c44c5490a9ea9a7653f0e7_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,477 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bb447704a55947398bc9729aa3cf7bea_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bb447704a55947398bc9729aa3cf7bea_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,478 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bc80fdb2032a457dbf0e7d559d63d3f9_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bc80fdb2032a457dbf0e7d559d63d3f9_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,479 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119be85d1ab30984896806fcab91d06a707_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119be85d1ab30984896806fcab91d06a707_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,481 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c297d2a46ea74d2a8058f62de9b5bd42_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c297d2a46ea74d2a8058f62de9b5bd42_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,482 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c738766cf2544c2f95450579b6ba9adf_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c738766cf2544c2f95450579b6ba9adf_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,483 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e7d077fff9c9430c923b603778b82f4c_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e7d077fff9c9430c923b603778b82f4c_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,484 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fcd7208ae0f443e59a7ce668ef909ab2_f012bfcce8bd6aef12a49ca6ef302f96 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fcd7208ae0f443e59a7ce668ef909ab2_f012bfcce8bd6aef12a49ca6ef302f96 2024-11-19T15:14:44,485 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T15:14:44,487 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:44,489 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T15:14:44,491 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T15:14:44,492 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:44,492 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T15:14:44,493 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732029284492"}]},"ts":"9223372036854775807"} 2024-11-19T15:14:44,494 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T15:14:44,495 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f012bfcce8bd6aef12a49ca6ef302f96, NAME => 'TestAcidGuarantees,,1732029249350.f012bfcce8bd6aef12a49ca6ef302f96.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T15:14:44,495 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T15:14:44,495 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732029284495"}]},"ts":"9223372036854775807"} 2024-11-19T15:14:44,497 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T15:14:44,497 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:44,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-19T15:14:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T15:14:44,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T15:14:44,545 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:44,559 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247 (was 248), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=482 (was 360) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8237 (was 8577) 2024-11-19T15:14:44,569 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=482, ProcessCount=11, AvailableMemoryMB=8236 2024-11-19T15:14:44,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T15:14:44,570 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T15:14:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T15:14:44,572 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T15:14:44,573 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:44,573 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-19T15:14:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-19T15:14:44,574 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T15:14:44,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742065_1241 (size=1061) 2024-11-19T15:14:44,582 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8 2024-11-19T15:14:44,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742066_1242 (size=53) 2024-11-19T15:14:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-19T15:14:44,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-19T15:14:44,993 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:14:44,993 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bd5712f88c144f357fc59afd6d3ab798, disabling compactions & flushes 2024-11-19T15:14:44,993 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:44,993 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:44,993 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. after waiting 0 ms 2024-11-19T15:14:44,993 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:44,993 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:44,993 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bd5712f88c144f357fc59afd6d3ab798: Waiting for close lock at 1732029284993Disabling compacts and flushes for region at 1732029284993Disabling writes for close at 1732029284993Writing region close event to WAL at 1732029284993Closed at 1732029284993 2024-11-19T15:14:44,994 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T15:14:44,994 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732029284994"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732029284994"}]},"ts":"1732029284994"} 2024-11-19T15:14:44,996 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T15:14:44,997 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T15:14:44,997 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029284997"}]},"ts":"1732029284997"} 2024-11-19T15:14:44,999 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T15:14:44,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd5712f88c144f357fc59afd6d3ab798, ASSIGN}] 2024-11-19T15:14:45,000 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd5712f88c144f357fc59afd6d3ab798, ASSIGN 2024-11-19T15:14:45,000 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd5712f88c144f357fc59afd6d3ab798, ASSIGN; state=OFFLINE, location=50570b5b74e7,46557,1732029218639; forceNewPlan=false, retain=false 2024-11-19T15:14:45,151 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=bd5712f88c144f357fc59afd6d3ab798, regionState=OPENING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:45,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd5712f88c144f357fc59afd6d3ab798, ASSIGN because future has completed 2024-11-19T15:14:45,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:14:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-19T15:14:45,308 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:45,308 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} 2024-11-19T15:14:45,309 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,309 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:14:45,309 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,309 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,311 INFO [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,312 INFO [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:14:45,313 INFO [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd5712f88c144f357fc59afd6d3ab798 columnFamilyName A 2024-11-19T15:14:45,313 DEBUG [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:45,313 INFO [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] regionserver.HStore(327): Store=bd5712f88c144f357fc59afd6d3ab798/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:14:45,313 INFO [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,314 INFO [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:14:45,314 INFO [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd5712f88c144f357fc59afd6d3ab798 columnFamilyName B 2024-11-19T15:14:45,314 DEBUG [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:45,315 INFO [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] regionserver.HStore(327): Store=bd5712f88c144f357fc59afd6d3ab798/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:14:45,315 INFO [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,315 INFO [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:14:45,315 INFO [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd5712f88c144f357fc59afd6d3ab798 columnFamilyName C 2024-11-19T15:14:45,316 DEBUG [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:14:45,316 INFO [StoreOpener-bd5712f88c144f357fc59afd6d3ab798-1 {}] regionserver.HStore(327): Store=bd5712f88c144f357fc59afd6d3ab798/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:14:45,316 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:45,316 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,317 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,317 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,318 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,318 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,318 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T15:14:45,319 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,321 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T15:14:45,322 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened bd5712f88c144f357fc59afd6d3ab798; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59860739, jitterRate=-0.10800547897815704}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T15:14:45,322 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,323 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for bd5712f88c144f357fc59afd6d3ab798: Running coprocessor pre-open hook at 1732029285309Writing region info on filesystem at 1732029285309Initializing all the Stores at 1732029285310 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029285310Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029285311 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029285311Cleaning up temporary data from old regions at 1732029285318 (+7 ms)Running coprocessor post-open hooks at 1732029285322 (+4 ms)Region opened successfully at 1732029285323 (+1 ms) 2024-11-19T15:14:45,323 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., pid=63, masterSystemTime=1732029285305 2024-11-19T15:14:45,325 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:45,325 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:45,326 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=bd5712f88c144f357fc59afd6d3ab798, regionState=OPEN, openSeqNum=2, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:45,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:14:45,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-19T15:14:45,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 in 175 msec 2024-11-19T15:14:45,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-19T15:14:45,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd5712f88c144f357fc59afd6d3ab798, ASSIGN in 331 msec 2024-11-19T15:14:45,333 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T15:14:45,333 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029285333"}]},"ts":"1732029285333"} 2024-11-19T15:14:45,334 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T15:14:45,335 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T15:14:45,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 765 msec 2024-11-19T15:14:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-19T15:14:45,714 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:45,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,715 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:45,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:45,716 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb0c4fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:45,719 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:45,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54eed65f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,720 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:45,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:45,722 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T15:14:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 50570b5b74e7,39501,1732029217890 2024-11-19T15:14:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67f00197 2024-11-19T15:14:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T15:14:45,724 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T15:14:45,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:45,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:45,726 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c959694, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:45,727 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:45,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d558122, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,729 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:45,729 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:45,729 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@933a0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:45,730 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14a1db3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:45,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:45,732 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eebc47c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:45,733 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f148dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,734 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:45,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:45,735 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f9c8ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:45,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219c4075, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,739 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:45,739 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:45,740 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d2a3b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:45,741 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ecbf0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:45,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:45,743 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce38946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:45,744 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b75fee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:45,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:45,746 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49d78a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:45,747 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf87ab3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:45,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:45,749 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b97574, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:45,751 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f53191c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:45,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:45,753 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6327c707, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:45,754 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4537d611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72a925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:14:45,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:14:45,756 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:14:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:14:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:14:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e10012d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:14:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:14:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:14:45,758 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:14:45,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46199b51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:14:45,759 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:45,759 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:45,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:45,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:45,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:45,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:45,760 DEBUG [Thread-1139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:45,760 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:45,761 DEBUG [Thread-1139 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:45,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:45,761 DEBUG [Thread-1139 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:45,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:45,762 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:45,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:45,763 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:45,763 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:45,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:45,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:45,764 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-19T15:14:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:14:45,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T15:14:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:14:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:45,766 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:14:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:14:45,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:14:45,768 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:45,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:14:45,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:45,770 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:45,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:14:45,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:14:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:14:45,776 DEBUG [Thread-1141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:14:45,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:14:45,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:14:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:45,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:14:45,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:14:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:14:45,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:45,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:45,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/b31654aef239421daae93e3a475b336b is 50, key is test_row_0/A:col10/1732029285771/Put/seqid=0 2024-11-19T15:14:45,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732029285821, max timestamp=1732029285833Num uniques 859; 2024-11-19T15:14:45,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T15:14:45,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029285823, max timestamp=1732029285833Num uniques 859; 2024-11-19T15:14:45,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:45,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029285823, max timestamp=1732029285833Num uniques 859; 2024-11-19T15:14:45,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:45,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742067_1243 (size=12001) 2024-11-19T15:14:45,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/b31654aef239421daae93e3a475b336b 2024-11-19T15:14:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T15:14:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029345862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:45,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029345866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:45,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37398 deadline: 1732029345871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:45,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/596dbd0d1d8a48ddbedf1f35b8363ad2 is 50, key is test_row_0/B:col10/1732029285771/Put/seqid=0 2024-11-19T15:14:45,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37312 deadline: 1732029345877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:45,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37386 deadline: 1732029345878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:45,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T15:14:45,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:45,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:45,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:45,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:45,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:45,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742068_1244 (size=12001) 2024-11-19T15:14:45,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/596dbd0d1d8a48ddbedf1f35b8363ad2 2024-11-19T15:14:45,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/8849575653a54682836ca065b4f631b4 is 50, key is test_row_0/C:col10/1732029285771/Put/seqid=0 2024-11-19T15:14:45,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029345995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:45,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742069_1245 (size=12001) 2024-11-19T15:14:45,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:45,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:45,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/8849575653a54682836ca065b4f631b4 2024-11-19T15:14:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029345995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37312 deadline: 1732029345995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/b31654aef239421daae93e3a475b336b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b31654aef239421daae93e3a475b336b 2024-11-19T15:14:46,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b31654aef239421daae93e3a475b336b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T15:14:46,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37386 deadline: 1732029346004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/596dbd0d1d8a48ddbedf1f35b8363ad2 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/596dbd0d1d8a48ddbedf1f35b8363ad2 2024-11-19T15:14:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37398 deadline: 1732029346004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/596dbd0d1d8a48ddbedf1f35b8363ad2, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T15:14:46,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/8849575653a54682836ca065b4f631b4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/8849575653a54682836ca065b4f631b4 2024-11-19T15:14:46,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/8849575653a54682836ca065b4f631b4, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T15:14:46,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=129.71 KB/132820 for bd5712f88c144f357fc59afd6d3ab798 in 257ms, sequenceid=13, compaction requested=false 2024-11-19T15:14:46,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:46,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T15:14:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:46,075 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-11-19T15:14:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T15:14:46,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e00bfbbc8488461a9b4666ebcd0b6616 is 50, key is test_row_0/A:col10/1732029285860/Put/seqid=0 2024-11-19T15:14:46,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742070_1246 (size=12001) 2024-11-19T15:14:46,113 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e00bfbbc8488461a9b4666ebcd0b6616 2024-11-19T15:14:46,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/63fb6d90f3214e0f9cfdb309f56694d5 is 50, key is test_row_0/B:col10/1732029285860/Put/seqid=0 2024-11-19T15:14:46,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742071_1247 (size=12001) 2024-11-19T15:14:46,154 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/63fb6d90f3214e0f9cfdb309f56694d5 2024-11-19T15:14:46,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d581c816a4f847c5bb74cbc5fcf11743 is 50, key is test_row_0/C:col10/1732029285860/Put/seqid=0 2024-11-19T15:14:46,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742072_1248 (size=12001) 2024-11-19T15:14:46,211 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d581c816a4f847c5bb74cbc5fcf11743 2024-11-19T15:14:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:46,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:46,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e00bfbbc8488461a9b4666ebcd0b6616 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e00bfbbc8488461a9b4666ebcd0b6616 2024-11-19T15:14:46,225 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e00bfbbc8488461a9b4666ebcd0b6616, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T15:14:46,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/63fb6d90f3214e0f9cfdb309f56694d5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/63fb6d90f3214e0f9cfdb309f56694d5 2024-11-19T15:14:46,238 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/63fb6d90f3214e0f9cfdb309f56694d5, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T15:14:46,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029346226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029346230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d581c816a4f847c5bb74cbc5fcf11743 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d581c816a4f847c5bb74cbc5fcf11743 2024-11-19T15:14:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,246 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d581c816a4f847c5bb74cbc5fcf11743, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T15:14:46,247 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~365.41 KB/374176, currentSize=60.38 KB/61830 for bd5712f88c144f357fc59afd6d3ab798 in 172ms, sequenceid=48, compaction requested=false 2024-11-19T15:14:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-19T15:14:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-19T15:14:46,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T15:14:46,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:46,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:46,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:46,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:46,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:46,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:46,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-19T15:14:46,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 482 msec 2024-11-19T15:14:46,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 491 msec 2024-11-19T15:14:46,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/beb4271011f542e7a6436a7218b8a68e is 50, key is test_row_0/A:col10/1732029286249/Put/seqid=0 2024-11-19T15:14:46,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742073_1249 (size=12001) 2024-11-19T15:14:46,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37312 deadline: 1732029346333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37398 deadline: 1732029346339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37386 deadline: 1732029346340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029346344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029346344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T15:14:46,395 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:46,395 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-19T15:14:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T15:14:46,397 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:46,398 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:46,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:46,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37312 deadline: 1732029346444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37386 deadline: 1732029346455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37398 deadline: 1732029346455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T15:14:46,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T15:14:46,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:46,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:46,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:46,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:46,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:46,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029346555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029346555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37312 deadline: 1732029346654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37386 deadline: 1732029346664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37398 deadline: 1732029346664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/beb4271011f542e7a6436a7218b8a68e 2024-11-19T15:14:46,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/4fe115ca0b4848c48a1a90010a807964 is 50, key is test_row_0/B:col10/1732029286249/Put/seqid=0 2024-11-19T15:14:46,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742074_1250 (size=12001) 2024-11-19T15:14:46,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/4fe115ca0b4848c48a1a90010a807964 2024-11-19T15:14:46,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T15:14:46,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:46,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:46,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:46,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:46,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T15:14:46,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/1403f48dbad54d8aad16fb1dd2ac7937 is 50, key is test_row_0/C:col10/1732029286249/Put/seqid=0 2024-11-19T15:14:46,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742075_1251 (size=12001) 2024-11-19T15:14:46,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T15:14:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:46,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:46,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029346864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029346864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37312 deadline: 1732029346964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37398 deadline: 1732029346974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37386 deadline: 1732029346974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:47,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T15:14:47,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:47,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T15:14:47,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/1403f48dbad54d8aad16fb1dd2ac7937 2024-11-19T15:14:47,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/beb4271011f542e7a6436a7218b8a68e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/beb4271011f542e7a6436a7218b8a68e 2024-11-19T15:14:47,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/beb4271011f542e7a6436a7218b8a68e, entries=150, sequenceid=61, filesize=11.7 K 2024-11-19T15:14:47,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/4fe115ca0b4848c48a1a90010a807964 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4fe115ca0b4848c48a1a90010a807964 2024-11-19T15:14:47,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4fe115ca0b4848c48a1a90010a807964, entries=150, sequenceid=61, filesize=11.7 K 2024-11-19T15:14:47,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/1403f48dbad54d8aad16fb1dd2ac7937 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/1403f48dbad54d8aad16fb1dd2ac7937 2024-11-19T15:14:47,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T15:14:47,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:47,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:47,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:47,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:47,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:47,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/1403f48dbad54d8aad16fb1dd2ac7937, entries=150, sequenceid=61, filesize=11.7 K 2024-11-19T15:14:47,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for bd5712f88c144f357fc59afd6d3ab798 in 922ms, sequenceid=61, compaction requested=true 2024-11-19T15:14:47,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:47,173 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:47,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:47,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:47,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:47,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:47,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:47,173 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:47,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:47,175 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:47,175 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/B is initiating minor compaction (all files) 2024-11-19T15:14:47,175 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/B in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:47,175 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/596dbd0d1d8a48ddbedf1f35b8363ad2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/63fb6d90f3214e0f9cfdb309f56694d5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4fe115ca0b4848c48a1a90010a807964] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=35.2 K 2024-11-19T15:14:47,175 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:47,175 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/A is initiating minor compaction (all files) 2024-11-19T15:14:47,175 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/A in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:47,176 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 596dbd0d1d8a48ddbedf1f35b8363ad2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732029285771 2024-11-19T15:14:47,176 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b31654aef239421daae93e3a475b336b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e00bfbbc8488461a9b4666ebcd0b6616, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/beb4271011f542e7a6436a7218b8a68e] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=35.2 K 2024-11-19T15:14:47,176 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 63fb6d90f3214e0f9cfdb309f56694d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732029285860 2024-11-19T15:14:47,177 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fe115ca0b4848c48a1a90010a807964, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732029286236 2024-11-19T15:14:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:47,177 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting b31654aef239421daae93e3a475b336b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732029285771 2024-11-19T15:14:47,178 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e00bfbbc8488461a9b4666ebcd0b6616, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732029285860 2024-11-19T15:14:47,179 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting beb4271011f542e7a6436a7218b8a68e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732029286236 2024-11-19T15:14:47,193 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#B#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:47,194 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/e5b6337c5cb144febb5c86cf7b033509 is 50, key is test_row_0/B:col10/1732029286249/Put/seqid=0 2024-11-19T15:14:47,200 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#A#compaction#208 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:47,200 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/cc9badbe797f44d2a414802b84fed854 is 50, key is test_row_0/A:col10/1732029286249/Put/seqid=0 2024-11-19T15:14:47,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742076_1252 (size=12104) 2024-11-19T15:14:47,236 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/e5b6337c5cb144febb5c86cf7b033509 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/e5b6337c5cb144febb5c86cf7b033509 2024-11-19T15:14:47,245 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/B of bd5712f88c144f357fc59afd6d3ab798 into e5b6337c5cb144febb5c86cf7b033509(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:47,245 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:47,245 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/B, priority=13, startTime=1732029287173; duration=0sec 2024-11-19T15:14:47,245 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:47,245 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:B 2024-11-19T15:14:47,246 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:47,248 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:47,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742077_1253 (size=12104) 2024-11-19T15:14:47,250 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/C is initiating minor compaction (all files) 2024-11-19T15:14:47,250 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/C in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:47,250 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/8849575653a54682836ca065b4f631b4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d581c816a4f847c5bb74cbc5fcf11743, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/1403f48dbad54d8aad16fb1dd2ac7937] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=35.2 K 2024-11-19T15:14:47,251 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8849575653a54682836ca065b4f631b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732029285771 2024-11-19T15:14:47,254 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d581c816a4f847c5bb74cbc5fcf11743, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732029285860 2024-11-19T15:14:47,255 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 1403f48dbad54d8aad16fb1dd2ac7937, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732029286236 2024-11-19T15:14:47,257 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/cc9badbe797f44d2a414802b84fed854 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/cc9badbe797f44d2a414802b84fed854 2024-11-19T15:14:47,268 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/A of bd5712f88c144f357fc59afd6d3ab798 into cc9badbe797f44d2a414802b84fed854(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:47,268 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:47,268 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/A, priority=13, startTime=1732029287172; duration=0sec 2024-11-19T15:14:47,268 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:47,268 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:A 2024-11-19T15:14:47,273 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#C#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:47,274 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/e82fa89591e541b49c722f44570a761e is 50, key is test_row_0/C:col10/1732029286249/Put/seqid=0 2024-11-19T15:14:47,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742078_1254 (size=12104) 2024-11-19T15:14:47,302 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/e82fa89591e541b49c722f44570a761e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/e82fa89591e541b49c722f44570a761e 2024-11-19T15:14:47,311 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/C of bd5712f88c144f357fc59afd6d3ab798 into e82fa89591e541b49c722f44570a761e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:47,311 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:47,311 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/C, priority=13, startTime=1732029287173; duration=0sec 2024-11-19T15:14:47,311 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:47,311 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:C 2024-11-19T15:14:47,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T15:14:47,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:47,330 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T15:14:47,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:47,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:47,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:47,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:47,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:47,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:47,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/3091c3eaaa234740af2e4f37bc5d2ac6 is 50, key is test_row_0/A:col10/1732029286338/Put/seqid=0 2024-11-19T15:14:47,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742079_1255 (size=12001) 2024-11-19T15:14:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:47,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:47,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029347401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:47,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029347406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:47,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:47,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37398 deadline: 1732029347485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37312 deadline: 1732029347485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:47,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37386 deadline: 1732029347485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:47,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029347519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029347519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T15:14:47,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029347724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:47,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029347724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:47,771 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/3091c3eaaa234740af2e4f37bc5d2ac6 2024-11-19T15:14:47,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/8a044c518bf44c9f96407e834722b070 is 50, key is test_row_0/B:col10/1732029286338/Put/seqid=0 2024-11-19T15:14:47,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742080_1256 (size=12001) 2024-11-19T15:14:48,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029348034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:48,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029348044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:48,234 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/8a044c518bf44c9f96407e834722b070 2024-11-19T15:14:48,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d6377693396e48d6a6833d54494d9ac1 is 50, key is test_row_0/C:col10/1732029286338/Put/seqid=0 2024-11-19T15:14:48,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742081_1257 (size=12001) 2024-11-19T15:14:48,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37312 deadline: 1732029348494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:48,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37386 deadline: 1732029348494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:48,496 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in bd5712f88c144f357fc59afd6d3ab798 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:48,497 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in bd5712f88c144f357fc59afd6d3ab798 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:48,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37398 deadline: 1732029348504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:48,505 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in bd5712f88c144f357fc59afd6d3ab798 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:48,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T15:14:48,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029348544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:48,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029348554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:48,649 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d6377693396e48d6a6833d54494d9ac1 2024-11-19T15:14:48,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/3091c3eaaa234740af2e4f37bc5d2ac6 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/3091c3eaaa234740af2e4f37bc5d2ac6 2024-11-19T15:14:48,657 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/3091c3eaaa234740af2e4f37bc5d2ac6, entries=150, sequenceid=84, filesize=11.7 K 2024-11-19T15:14:48,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/8a044c518bf44c9f96407e834722b070 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/8a044c518bf44c9f96407e834722b070 2024-11-19T15:14:48,662 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/8a044c518bf44c9f96407e834722b070, entries=150, sequenceid=84, filesize=11.7 K 2024-11-19T15:14:48,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d6377693396e48d6a6833d54494d9ac1 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d6377693396e48d6a6833d54494d9ac1 2024-11-19T15:14:48,667 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d6377693396e48d6a6833d54494d9ac1, entries=150, sequenceid=84, filesize=11.7 K 2024-11-19T15:14:48,668 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for bd5712f88c144f357fc59afd6d3ab798 in 1338ms, sequenceid=84, compaction requested=false 2024-11-19T15:14:48,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:48,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:48,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-19T15:14:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-19T15:14:48,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-19T15:14:48,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2700 sec 2024-11-19T15:14:48,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.2770 sec 2024-11-19T15:14:49,390 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T15:14:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:49,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T15:14:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:49,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:49,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/c358b38942bf411c9ef62a088ccb95f0 is 50, key is test_row_0/A:col10/1732029287399/Put/seqid=0 2024-11-19T15:14:49,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742082_1258 (size=14341) 2024-11-19T15:14:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029349606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:49,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029349608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:49,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:49,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:49,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:49,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029349714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:49,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029349715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:49,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:49,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:49,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029349924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:49,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029349924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:49,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/c358b38942bf411c9ef62a088ccb95f0 2024-11-19T15:14:49,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/99c1f3d2478041d2aa6e3a8c27ab429f is 50, key is test_row_0/B:col10/1732029287399/Put/seqid=0 2024-11-19T15:14:49,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742083_1259 (size=12001) 2024-11-19T15:14:50,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/99c1f3d2478041d2aa6e3a8c27ab429f 2024-11-19T15:14:50,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/a1939588f351404f8e046a9c92ae4bfe is 50, key is test_row_0/C:col10/1732029287399/Put/seqid=0 2024-11-19T15:14:50,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742084_1260 (size=12001) 2024-11-19T15:14:50,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/a1939588f351404f8e046a9c92ae4bfe 2024-11-19T15:14:50,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/c358b38942bf411c9ef62a088ccb95f0 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/c358b38942bf411c9ef62a088ccb95f0 2024-11-19T15:14:50,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/c358b38942bf411c9ef62a088ccb95f0, entries=200, sequenceid=98, filesize=14.0 K 2024-11-19T15:14:50,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/99c1f3d2478041d2aa6e3a8c27ab429f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/99c1f3d2478041d2aa6e3a8c27ab429f 2024-11-19T15:14:50,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/99c1f3d2478041d2aa6e3a8c27ab429f, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T15:14:50,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/a1939588f351404f8e046a9c92ae4bfe as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a1939588f351404f8e046a9c92ae4bfe 2024-11-19T15:14:50,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a1939588f351404f8e046a9c92ae4bfe, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T15:14:50,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for bd5712f88c144f357fc59afd6d3ab798 in 480ms, sequenceid=98, compaction requested=true 2024-11-19T15:14:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:50,048 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:50,048 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:50,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:50,049 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:50,049 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:50,049 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/B is initiating minor compaction (all files) 2024-11-19T15:14:50,049 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/A is initiating minor compaction (all files) 2024-11-19T15:14:50,049 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/B in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:50,049 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/A in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:50,049 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/e5b6337c5cb144febb5c86cf7b033509, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/8a044c518bf44c9f96407e834722b070, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/99c1f3d2478041d2aa6e3a8c27ab429f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=35.3 K 2024-11-19T15:14:50,049 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/cc9badbe797f44d2a414802b84fed854, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/3091c3eaaa234740af2e4f37bc5d2ac6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/c358b38942bf411c9ef62a088ccb95f0] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=37.5 K 2024-11-19T15:14:50,050 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting e5b6337c5cb144febb5c86cf7b033509, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732029286236 2024-11-19T15:14:50,050 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc9badbe797f44d2a414802b84fed854, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732029286236 2024-11-19T15:14:50,050 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a044c518bf44c9f96407e834722b070, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732029286322 2024-11-19T15:14:50,050 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3091c3eaaa234740af2e4f37bc5d2ac6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732029286322 2024-11-19T15:14:50,051 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 99c1f3d2478041d2aa6e3a8c27ab429f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732029287377 2024-11-19T15:14:50,051 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting c358b38942bf411c9ef62a088ccb95f0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732029287377 2024-11-19T15:14:50,059 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#B#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:50,060 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/57ff9c24db9247a6a5badb3451569775 is 50, key is test_row_0/B:col10/1732029287399/Put/seqid=0 2024-11-19T15:14:50,060 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#A#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:50,061 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/72f272c1bef743afa51d9f11d3b29357 is 50, key is test_row_0/A:col10/1732029287399/Put/seqid=0 2024-11-19T15:14:50,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742085_1261 (size=12207) 2024-11-19T15:14:50,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742086_1262 (size=12207) 2024-11-19T15:14:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:50,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T15:14:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:50,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/0f1b46dd8a4042caa2b7b4d6b346df54 is 50, key is test_row_0/A:col10/1732029290236/Put/seqid=0 2024-11-19T15:14:50,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742087_1263 (size=12001) 2024-11-19T15:14:50,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/0f1b46dd8a4042caa2b7b4d6b346df54 2024-11-19T15:14:50,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/7802b3e03b8b45bd972efa9f6b34ac01 is 50, key is test_row_0/B:col10/1732029290236/Put/seqid=0 2024-11-19T15:14:50,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742088_1264 (size=12001) 2024-11-19T15:14:50,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:50,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029350269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:50,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029350269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:50,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:50,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029350384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029350384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:50,480 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/57ff9c24db9247a6a5badb3451569775 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/57ff9c24db9247a6a5badb3451569775 2024-11-19T15:14:50,486 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/B of bd5712f88c144f357fc59afd6d3ab798 into 57ff9c24db9247a6a5badb3451569775(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:50,486 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:50,486 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/B, priority=13, startTime=1732029290048; duration=0sec 2024-11-19T15:14:50,486 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:50,486 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:B 2024-11-19T15:14:50,486 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:50,487 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/72f272c1bef743afa51d9f11d3b29357 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/72f272c1bef743afa51d9f11d3b29357 2024-11-19T15:14:50,488 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:50,488 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/C is initiating minor compaction (all files) 2024-11-19T15:14:50,488 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/C in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:50,488 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/e82fa89591e541b49c722f44570a761e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d6377693396e48d6a6833d54494d9ac1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a1939588f351404f8e046a9c92ae4bfe] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=35.3 K 2024-11-19T15:14:50,489 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting e82fa89591e541b49c722f44570a761e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732029286236 2024-11-19T15:14:50,489 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d6377693396e48d6a6833d54494d9ac1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732029286322 2024-11-19T15:14:50,490 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting a1939588f351404f8e046a9c92ae4bfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732029287377 2024-11-19T15:14:50,493 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/A of bd5712f88c144f357fc59afd6d3ab798 into 72f272c1bef743afa51d9f11d3b29357(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:50,493 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:50,493 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/A, priority=13, startTime=1732029290048; duration=0sec 2024-11-19T15:14:50,493 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:50,493 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:A 2024-11-19T15:14:50,499 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#C#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:50,500 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/7a109ee8a70942a0a99fc2cb2acf7e8c is 50, key is test_row_0/C:col10/1732029287399/Put/seqid=0 2024-11-19T15:14:50,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37386 deadline: 1732029350504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:50,510 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in bd5712f88c144f357fc59afd6d3ab798 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:50,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742089_1265 (size=12207) 2024-11-19T15:14:50,525 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/7a109ee8a70942a0a99fc2cb2acf7e8c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/7a109ee8a70942a0a99fc2cb2acf7e8c 2024-11-19T15:14:50,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37398 deadline: 1732029350524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:50,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37312 deadline: 1732029350524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:50,528 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in bd5712f88c144f357fc59afd6d3ab798 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:50,529 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in bd5712f88c144f357fc59afd6d3ab798 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:50,531 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/C of bd5712f88c144f357fc59afd6d3ab798 into 7a109ee8a70942a0a99fc2cb2acf7e8c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:50,531 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:50,531 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/C, priority=13, startTime=1732029290048; duration=0sec 2024-11-19T15:14:50,531 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:50,531 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:C 2024-11-19T15:14:50,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T15:14:50,575 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:50,575 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:50,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-19T15:14:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T15:14:50,577 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:50,578 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:50,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:50,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029350594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:50,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029350595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:50,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/7802b3e03b8b45bd972efa9f6b34ac01 2024-11-19T15:14:50,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/019bae089d8243a088609d6dcf03349d is 50, key is test_row_0/C:col10/1732029290236/Put/seqid=0 2024-11-19T15:14:50,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742090_1266 (size=12001) 2024-11-19T15:14:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T15:14:50,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T15:14:50,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:50,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:50,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:50,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:50,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:50,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T15:14:50,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:50,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:50,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:50,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:50,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T15:14:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029350904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029350904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:51,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T15:14:51,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:51,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:51,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:51,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:51,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:51,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/019bae089d8243a088609d6dcf03349d 2024-11-19T15:14:51,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/0f1b46dd8a4042caa2b7b4d6b346df54 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/0f1b46dd8a4042caa2b7b4d6b346df54 2024-11-19T15:14:51,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/0f1b46dd8a4042caa2b7b4d6b346df54, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T15:14:51,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/7802b3e03b8b45bd972efa9f6b34ac01 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/7802b3e03b8b45bd972efa9f6b34ac01 2024-11-19T15:14:51,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/7802b3e03b8b45bd972efa9f6b34ac01, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T15:14:51,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/019bae089d8243a088609d6dcf03349d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/019bae089d8243a088609d6dcf03349d 2024-11-19T15:14:51,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/019bae089d8243a088609d6dcf03349d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T15:14:51,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for bd5712f88c144f357fc59afd6d3ab798 in 856ms, sequenceid=117, compaction requested=false 2024-11-19T15:14:51,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:51,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T15:14:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:51,190 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T15:14:51,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:51,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:51,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:51,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:51,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:51,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:51,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/9c525f98cddb4e3eac1fd412f7f2b0c7 is 50, key is test_row_0/A:col10/1732029290247/Put/seqid=0 2024-11-19T15:14:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T15:14:51,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742091_1267 (size=12051) 2024-11-19T15:14:51,207 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/9c525f98cddb4e3eac1fd412f7f2b0c7 2024-11-19T15:14:51,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/674400ff4d5b4c4f82be917ea92cb768 is 50, key is test_row_0/B:col10/1732029290247/Put/seqid=0 2024-11-19T15:14:51,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742092_1268 (size=12051) 2024-11-19T15:14:51,221 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/674400ff4d5b4c4f82be917ea92cb768 2024-11-19T15:14:51,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/556b6ae71a3e47a19e9e3179a253f1f7 is 50, key is test_row_0/C:col10/1732029290247/Put/seqid=0 2024-11-19T15:14:51,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742093_1269 (size=12051) 2024-11-19T15:14:51,239 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/556b6ae71a3e47a19e9e3179a253f1f7 2024-11-19T15:14:51,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/9c525f98cddb4e3eac1fd412f7f2b0c7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/9c525f98cddb4e3eac1fd412f7f2b0c7 2024-11-19T15:14:51,249 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/9c525f98cddb4e3eac1fd412f7f2b0c7, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T15:14:51,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/674400ff4d5b4c4f82be917ea92cb768 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/674400ff4d5b4c4f82be917ea92cb768 2024-11-19T15:14:51,254 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/674400ff4d5b4c4f82be917ea92cb768, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T15:14:51,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/556b6ae71a3e47a19e9e3179a253f1f7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/556b6ae71a3e47a19e9e3179a253f1f7 2024-11-19T15:14:51,259 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/556b6ae71a3e47a19e9e3179a253f1f7, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T15:14:51,260 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for bd5712f88c144f357fc59afd6d3ab798 in 70ms, sequenceid=134, compaction requested=true 2024-11-19T15:14:51,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:51,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:51,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-19T15:14:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-19T15:14:51,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-19T15:14:51,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 684 msec 2024-11-19T15:14:51,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 690 msec 2024-11-19T15:14:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:51,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:14:51,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:51,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:51,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:51,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:51,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:51,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:51,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/2c47e34c28aa4fe785c69ce6fdd6a5b5 is 50, key is test_row_0/A:col10/1732029291425/Put/seqid=0 2024-11-19T15:14:51,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742094_1270 (size=14541) 2024-11-19T15:14:51,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029291476, max timestamp=1732029291498Num uniques 859; 2024-11-19T15:14:51,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:51,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029291476, max timestamp=1732029291498Num uniques 859; 2024-11-19T15:14:51,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029291476, max timestamp=1732029291498Num uniques 859; 2024-11-19T15:14:51,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:51,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:51,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029351547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:51,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029351548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:51,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029351664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:51,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029351664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T15:14:51,714 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:51,715 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-19T15:14:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T15:14:51,717 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:51,718 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:51,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T15:14:51,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/2c47e34c28aa4fe785c69ce6fdd6a5b5 2024-11-19T15:14:51,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/0d0c7246cc9d4f6a85f7d746e8d3c32b is 50, key is test_row_0/B:col10/1732029291425/Put/seqid=0 2024-11-19T15:14:51,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742095_1271 (size=12151) 2024-11-19T15:14:51,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/0d0c7246cc9d4f6a85f7d746e8d3c32b 2024-11-19T15:14:51,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/9ad53a102aec4522ad1a2616a3a083a6 is 50, key is test_row_0/C:col10/1732029291425/Put/seqid=0 2024-11-19T15:14:51,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T15:14:51,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:51,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:51,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:51,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:51,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:51,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742096_1272 (size=12151) 2024-11-19T15:14:51,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029351874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:51,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029351874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:51,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:51,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:51,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:51,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:51,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:51,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:52,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T15:14:52,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:52,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:52,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:52,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:52,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:52,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T15:14:52,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T15:14:52,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:52,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:52,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:52,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:52,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:52,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029352184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:52,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029352184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:52,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/9ad53a102aec4522ad1a2616a3a083a6 2024-11-19T15:14:52,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/2c47e34c28aa4fe785c69ce6fdd6a5b5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/2c47e34c28aa4fe785c69ce6fdd6a5b5 2024-11-19T15:14:52,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/2c47e34c28aa4fe785c69ce6fdd6a5b5, entries=200, sequenceid=145, filesize=14.2 K 2024-11-19T15:14:52,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/0d0c7246cc9d4f6a85f7d746e8d3c32b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0d0c7246cc9d4f6a85f7d746e8d3c32b 2024-11-19T15:14:52,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0d0c7246cc9d4f6a85f7d746e8d3c32b, entries=150, sequenceid=145, filesize=11.9 K 2024-11-19T15:14:52,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/9ad53a102aec4522ad1a2616a3a083a6 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/9ad53a102aec4522ad1a2616a3a083a6 2024-11-19T15:14:52,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/9ad53a102aec4522ad1a2616a3a083a6, entries=150, sequenceid=145, filesize=11.9 K 2024-11-19T15:14:52,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bd5712f88c144f357fc59afd6d3ab798 in 871ms, sequenceid=145, compaction requested=true 2024-11-19T15:14:52,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:52,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:52,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:52,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:52,298 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:14:52,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:52,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:52,298 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:14:52,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:52,299 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:14:52,299 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/B is initiating minor compaction (all files) 2024-11-19T15:14:52,299 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/B in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:52,299 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/57ff9c24db9247a6a5badb3451569775, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/7802b3e03b8b45bd972efa9f6b34ac01, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/674400ff4d5b4c4f82be917ea92cb768, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0d0c7246cc9d4f6a85f7d746e8d3c32b] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=47.3 K 2024-11-19T15:14:52,300 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:14:52,300 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/A is initiating minor compaction (all files) 2024-11-19T15:14:52,300 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/A in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:52,300 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ff9c24db9247a6a5badb3451569775, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732029287377 2024-11-19T15:14:52,300 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/72f272c1bef743afa51d9f11d3b29357, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/0f1b46dd8a4042caa2b7b4d6b346df54, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/9c525f98cddb4e3eac1fd412f7f2b0c7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/2c47e34c28aa4fe785c69ce6fdd6a5b5] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=49.6 K 2024-11-19T15:14:52,300 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7802b3e03b8b45bd972efa9f6b34ac01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732029289600 2024-11-19T15:14:52,300 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72f272c1bef743afa51d9f11d3b29357, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732029287377 2024-11-19T15:14:52,301 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 674400ff4d5b4c4f82be917ea92cb768, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732029290247 2024-11-19T15:14:52,301 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f1b46dd8a4042caa2b7b4d6b346df54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732029289600 2024-11-19T15:14:52,301 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d0c7246cc9d4f6a85f7d746e8d3c32b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732029291420 2024-11-19T15:14:52,301 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c525f98cddb4e3eac1fd412f7f2b0c7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732029290247 2024-11-19T15:14:52,301 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c47e34c28aa4fe785c69ce6fdd6a5b5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732029291420 2024-11-19T15:14:52,309 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:52,310 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#B#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:52,310 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/9098e7d94b0b47a3891a5732d38e2c43 is 50, key is test_row_0/B:col10/1732029291425/Put/seqid=0 2024-11-19T15:14:52,310 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/de1726cea1ce4b56b5113014ea2d96a1 is 50, key is test_row_0/A:col10/1732029291425/Put/seqid=0 2024-11-19T15:14:52,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742098_1274 (size=12493) 2024-11-19T15:14:52,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742097_1273 (size=12493) 2024-11-19T15:14:52,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T15:14:52,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:52,335 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T15:14:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:52,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e0d9cfa17c5c4f549003bf18244b847f is 50, key is test_row_0/A:col10/1732029291538/Put/seqid=0 2024-11-19T15:14:52,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742099_1275 (size=12151) 2024-11-19T15:14:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T15:14:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:52,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:52,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029352715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:52,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029352718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:52,728 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/9098e7d94b0b47a3891a5732d38e2c43 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/9098e7d94b0b47a3891a5732d38e2c43 2024-11-19T15:14:52,730 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/de1726cea1ce4b56b5113014ea2d96a1 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/de1726cea1ce4b56b5113014ea2d96a1 2024-11-19T15:14:52,736 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/A of bd5712f88c144f357fc59afd6d3ab798 into de1726cea1ce4b56b5113014ea2d96a1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:52,736 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:52,736 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/A, priority=12, startTime=1732029292297; duration=0sec 2024-11-19T15:14:52,736 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:52,736 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:A 2024-11-19T15:14:52,736 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:14:52,737 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:14:52,737 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/C is initiating minor compaction (all files) 2024-11-19T15:14:52,737 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/C in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:52,737 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/7a109ee8a70942a0a99fc2cb2acf7e8c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/019bae089d8243a088609d6dcf03349d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/556b6ae71a3e47a19e9e3179a253f1f7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/9ad53a102aec4522ad1a2616a3a083a6] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=47.3 K 2024-11-19T15:14:52,738 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a109ee8a70942a0a99fc2cb2acf7e8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732029287377 2024-11-19T15:14:52,738 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 019bae089d8243a088609d6dcf03349d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732029289600 2024-11-19T15:14:52,739 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 556b6ae71a3e47a19e9e3179a253f1f7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732029290247 2024-11-19T15:14:52,739 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad53a102aec4522ad1a2616a3a083a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732029291420 2024-11-19T15:14:52,744 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e0d9cfa17c5c4f549003bf18244b847f 2024-11-19T15:14:52,751 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/B of bd5712f88c144f357fc59afd6d3ab798 into 9098e7d94b0b47a3891a5732d38e2c43(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:52,751 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:52,751 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#C#compaction#231 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:52,751 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/B, priority=12, startTime=1732029292298; duration=0sec 2024-11-19T15:14:52,751 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:52,751 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:B 2024-11-19T15:14:52,752 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/59560d9455af49848b98e45cb0ec66af is 50, key is test_row_0/C:col10/1732029291425/Put/seqid=0 2024-11-19T15:14:52,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/ade8df61c11d49df8bd909ddbbb2e8c2 is 50, key is test_row_0/B:col10/1732029291538/Put/seqid=0 2024-11-19T15:14:52,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742100_1276 (size=12493) 2024-11-19T15:14:52,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742101_1277 (size=12151) 2024-11-19T15:14:52,774 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/ade8df61c11d49df8bd909ddbbb2e8c2 2024-11-19T15:14:52,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/893c16256cff4339a63343b7ed6580ab is 50, key is test_row_0/C:col10/1732029291538/Put/seqid=0 2024-11-19T15:14:52,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742102_1278 (size=12151) 2024-11-19T15:14:52,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029352824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029352834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T15:14:53,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029353034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:53,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:53,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029353044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:53,169 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/59560d9455af49848b98e45cb0ec66af as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/59560d9455af49848b98e45cb0ec66af 2024-11-19T15:14:53,174 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/C of bd5712f88c144f357fc59afd6d3ab798 into 59560d9455af49848b98e45cb0ec66af(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:53,174 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:53,174 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/C, priority=12, startTime=1732029292298; duration=0sec 2024-11-19T15:14:53,174 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:53,174 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:C 2024-11-19T15:14:53,189 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/893c16256cff4339a63343b7ed6580ab 2024-11-19T15:14:53,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e0d9cfa17c5c4f549003bf18244b847f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e0d9cfa17c5c4f549003bf18244b847f 2024-11-19T15:14:53,197 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e0d9cfa17c5c4f549003bf18244b847f, entries=150, sequenceid=180, filesize=11.9 K 2024-11-19T15:14:53,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/ade8df61c11d49df8bd909ddbbb2e8c2 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/ade8df61c11d49df8bd909ddbbb2e8c2 2024-11-19T15:14:53,202 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/ade8df61c11d49df8bd909ddbbb2e8c2, entries=150, sequenceid=180, filesize=11.9 K 2024-11-19T15:14:53,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/893c16256cff4339a63343b7ed6580ab as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/893c16256cff4339a63343b7ed6580ab 2024-11-19T15:14:53,206 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/893c16256cff4339a63343b7ed6580ab, entries=150, sequenceid=180, filesize=11.9 K 2024-11-19T15:14:53,207 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for bd5712f88c144f357fc59afd6d3ab798 in 873ms, sequenceid=180, compaction requested=false 2024-11-19T15:14:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-19T15:14:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-19T15:14:53,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-19T15:14:53,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4910 sec 2024-11-19T15:14:53,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4960 sec 2024-11-19T15:14:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:53,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T15:14:53,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:53,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:53,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:53,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:53,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:53,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:53,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/373a470682df4992bb2d301a120572ea is 50, key is test_row_0/A:col10/1732029292717/Put/seqid=0 2024-11-19T15:14:53,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742103_1279 (size=14541) 2024-11-19T15:14:53,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029353393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:53,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029353399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:53,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029353504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:53,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029353514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:53,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:53,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029353714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:53,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029353724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:53,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/373a470682df4992bb2d301a120572ea 2024-11-19T15:14:53,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/918841a0bf4e46808d62fe22ffea1dea is 50, key is test_row_0/B:col10/1732029292717/Put/seqid=0 2024-11-19T15:14:53,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742104_1280 (size=12151) 2024-11-19T15:14:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T15:14:53,864 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:53,865 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-19T15:14:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T15:14:53,867 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:53,867 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:53,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T15:14:54,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T15:14:54,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:54,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:54,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:54,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:54,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:54,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029354024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:54,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029354034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:54,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/918841a0bf4e46808d62fe22ffea1dea 2024-11-19T15:14:54,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T15:14:54,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:54,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:54,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:54,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:54,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:54,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/18e903aacbdb436199ab597e3f765b8f is 50, key is test_row_0/C:col10/1732029292717/Put/seqid=0 2024-11-19T15:14:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T15:14:54,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742105_1281 (size=12151) 2024-11-19T15:14:54,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T15:14:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:54,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:54,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T15:14:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:54,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:54,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T15:14:54,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37386 deadline: 1732029354515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:54,519 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in bd5712f88c144f357fc59afd6d3ab798 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:54,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029354534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:54,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029354544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:54,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37312 deadline: 1732029354554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:54,557 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in bd5712f88c144f357fc59afd6d3ab798 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:54,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37398 deadline: 1732029354574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:54,578 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in bd5712f88c144f357fc59afd6d3ab798 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:14:54,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:54,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:54,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/18e903aacbdb436199ab597e3f765b8f 2024-11-19T15:14:54,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/373a470682df4992bb2d301a120572ea as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/373a470682df4992bb2d301a120572ea 2024-11-19T15:14:54,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/373a470682df4992bb2d301a120572ea, entries=200, sequenceid=195, filesize=14.2 K 2024-11-19T15:14:54,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/918841a0bf4e46808d62fe22ffea1dea as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/918841a0bf4e46808d62fe22ffea1dea 2024-11-19T15:14:54,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/918841a0bf4e46808d62fe22ffea1dea, entries=150, sequenceid=195, filesize=11.9 K 2024-11-19T15:14:54,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/18e903aacbdb436199ab597e3f765b8f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/18e903aacbdb436199ab597e3f765b8f 2024-11-19T15:14:54,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/18e903aacbdb436199ab597e3f765b8f, entries=150, sequenceid=195, filesize=11.9 K 2024-11-19T15:14:54,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for bd5712f88c144f357fc59afd6d3ab798 in 1276ms, sequenceid=195, compaction requested=true 2024-11-19T15:14:54,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:54,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:54,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:54,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:54,624 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:54,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:54,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:54,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:54,624 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:54,625 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:54,625 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/A is initiating minor compaction (all files) 2024-11-19T15:14:54,625 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/A in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:54,625 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/de1726cea1ce4b56b5113014ea2d96a1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e0d9cfa17c5c4f549003bf18244b847f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/373a470682df4992bb2d301a120572ea] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=38.3 K 2024-11-19T15:14:54,626 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:54,626 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/B is initiating minor compaction (all files) 2024-11-19T15:14:54,626 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting de1726cea1ce4b56b5113014ea2d96a1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732029291420 2024-11-19T15:14:54,626 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/B in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:54,626 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/9098e7d94b0b47a3891a5732d38e2c43, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/ade8df61c11d49df8bd909ddbbb2e8c2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/918841a0bf4e46808d62fe22ffea1dea] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=35.9 K 2024-11-19T15:14:54,626 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0d9cfa17c5c4f549003bf18244b847f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732029291537 2024-11-19T15:14:54,627 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 9098e7d94b0b47a3891a5732d38e2c43, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732029291420 2024-11-19T15:14:54,627 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 373a470682df4992bb2d301a120572ea, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732029292708 2024-11-19T15:14:54,627 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting ade8df61c11d49df8bd909ddbbb2e8c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732029291537 2024-11-19T15:14:54,627 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 918841a0bf4e46808d62fe22ffea1dea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732029292708 2024-11-19T15:14:54,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T15:14:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:54,634 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T15:14:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:54,639 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:54,640 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/c2e5c583d85148908ac94f62bc25e391 is 50, key is test_row_0/A:col10/1732029292717/Put/seqid=0 2024-11-19T15:14:54,647 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:54,647 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/0cda22b4dc904ab19e3fb86059b96c13 is 50, key is test_row_0/B:col10/1732029292717/Put/seqid=0 2024-11-19T15:14:54,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/b8909d92f1fe4b159735d3c3deeb1a3c is 50, key is test_row_0/A:col10/1732029293384/Put/seqid=0 2024-11-19T15:14:54,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742107_1283 (size=12595) 2024-11-19T15:14:54,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742106_1282 (size=12595) 2024-11-19T15:14:54,679 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/c2e5c583d85148908ac94f62bc25e391 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/c2e5c583d85148908ac94f62bc25e391 2024-11-19T15:14:54,684 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/A of bd5712f88c144f357fc59afd6d3ab798 into c2e5c583d85148908ac94f62bc25e391(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:54,684 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:54,684 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/A, priority=13, startTime=1732029294623; duration=0sec 2024-11-19T15:14:54,684 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:54,684 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:A 2024-11-19T15:14:54,685 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:54,685 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:54,686 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/C is initiating minor compaction (all files) 2024-11-19T15:14:54,686 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/C in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:54,686 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/59560d9455af49848b98e45cb0ec66af, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/893c16256cff4339a63343b7ed6580ab, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/18e903aacbdb436199ab597e3f765b8f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=35.9 K 2024-11-19T15:14:54,686 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59560d9455af49848b98e45cb0ec66af, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732029291420 2024-11-19T15:14:54,687 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 893c16256cff4339a63343b7ed6580ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732029291537 2024-11-19T15:14:54,687 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18e903aacbdb436199ab597e3f765b8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732029292708 2024-11-19T15:14:54,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742108_1284 (size=12151) 2024-11-19T15:14:54,696 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#C#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:54,697 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/44f099879c174d2a8c9d95baa8a58604 is 50, key is test_row_0/C:col10/1732029292717/Put/seqid=0 2024-11-19T15:14:54,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742109_1285 (size=12595) 2024-11-19T15:14:54,706 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/44f099879c174d2a8c9d95baa8a58604 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/44f099879c174d2a8c9d95baa8a58604 2024-11-19T15:14:54,712 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/C of bd5712f88c144f357fc59afd6d3ab798 into 44f099879c174d2a8c9d95baa8a58604(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:54,712 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:54,712 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/C, priority=13, startTime=1732029294624; duration=0sec 2024-11-19T15:14:54,712 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:54,712 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:C 2024-11-19T15:14:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T15:14:55,076 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/0cda22b4dc904ab19e3fb86059b96c13 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0cda22b4dc904ab19e3fb86059b96c13 2024-11-19T15:14:55,081 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/B of bd5712f88c144f357fc59afd6d3ab798 into 0cda22b4dc904ab19e3fb86059b96c13(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:55,081 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:55,081 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/B, priority=13, startTime=1732029294624; duration=0sec 2024-11-19T15:14:55,081 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:55,081 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:B 2024-11-19T15:14:55,094 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/b8909d92f1fe4b159735d3c3deeb1a3c 2024-11-19T15:14:55,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/b6b71dc088ab43bfb6033b557ef399ca is 50, key is test_row_0/B:col10/1732029293384/Put/seqid=0 2024-11-19T15:14:55,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742110_1286 (size=12151) 2024-11-19T15:14:55,109 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/b6b71dc088ab43bfb6033b557ef399ca 2024-11-19T15:14:55,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/4e8b6aa3eacb4d91bcc17b50d42e0457 is 50, key is test_row_0/C:col10/1732029293384/Put/seqid=0 2024-11-19T15:14:55,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742111_1287 (size=12151) 2024-11-19T15:14:55,522 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/4e8b6aa3eacb4d91bcc17b50d42e0457 2024-11-19T15:14:55,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/b8909d92f1fe4b159735d3c3deeb1a3c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b8909d92f1fe4b159735d3c3deeb1a3c 2024-11-19T15:14:55,530 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b8909d92f1fe4b159735d3c3deeb1a3c, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T15:14:55,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/b6b71dc088ab43bfb6033b557ef399ca as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/b6b71dc088ab43bfb6033b557ef399ca 2024-11-19T15:14:55,535 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/b6b71dc088ab43bfb6033b557ef399ca, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T15:14:55,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/4e8b6aa3eacb4d91bcc17b50d42e0457 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/4e8b6aa3eacb4d91bcc17b50d42e0457 2024-11-19T15:14:55,542 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/4e8b6aa3eacb4d91bcc17b50d42e0457, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T15:14:55,543 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for bd5712f88c144f357fc59afd6d3ab798 in 908ms, sequenceid=215, compaction requested=false 2024-11-19T15:14:55,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:55,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:55,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-19T15:14:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-19T15:14:55,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-19T15:14:55,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6780 sec 2024-11-19T15:14:55,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6840 sec 2024-11-19T15:14:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:55,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:14:55,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:55,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:55,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:55,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:55,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:55,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:55,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/4be5e23dc1b740b0913299ebaef8455b is 50, key is test_row_0/A:col10/1732029295572/Put/seqid=0 2024-11-19T15:14:55,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742112_1288 (size=12151) 2024-11-19T15:14:55,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029295607, max timestamp=1732029295622Num uniques 859; 2024-11-19T15:14:55,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029295607, max timestamp=1732029295622Num uniques 859; 2024-11-19T15:14:55,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029295607, max timestamp=1732029295622Num uniques 859; 2024-11-19T15:14:55,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:55,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:55,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:55,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029355670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:55,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029355671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:55,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029355784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029355784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:55,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/4be5e23dc1b740b0913299ebaef8455b 2024-11-19T15:14:55,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/1b432df1bf1143a2912701c375f4a4b5 is 50, key is test_row_0/B:col10/1732029295572/Put/seqid=0 2024-11-19T15:14:55,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029355995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:55,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029355995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:55,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742113_1289 (size=12151) 2024-11-19T15:14:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T15:14:56,014 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:14:56,015 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:14:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-19T15:14:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T15:14:56,017 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:14:56,018 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:14:56,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:14:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T15:14:56,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T15:14:56,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:56,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029356304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:56,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029356304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:56,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T15:14:56,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:56,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T15:14:56,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/1b432df1bf1143a2912701c375f4a4b5 2024-11-19T15:14:56,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/3786ed2918634132adcc8645cdc29eca is 50, key is test_row_0/C:col10/1732029295572/Put/seqid=0 2024-11-19T15:14:56,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742114_1290 (size=12151) 2024-11-19T15:14:56,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T15:14:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T15:14:56,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:56,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T15:14:56,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T15:14:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:14:56,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/3786ed2918634132adcc8645cdc29eca 2024-11-19T15:14:56,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/4be5e23dc1b740b0913299ebaef8455b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/4be5e23dc1b740b0913299ebaef8455b 2024-11-19T15:14:56,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029356814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:56,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029356814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:56,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/4be5e23dc1b740b0913299ebaef8455b, entries=150, sequenceid=228, filesize=11.9 K 2024-11-19T15:14:56,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/1b432df1bf1143a2912701c375f4a4b5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/1b432df1bf1143a2912701c375f4a4b5 2024-11-19T15:14:56,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/1b432df1bf1143a2912701c375f4a4b5, entries=150, sequenceid=228, filesize=11.9 K 2024-11-19T15:14:56,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/3786ed2918634132adcc8645cdc29eca as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/3786ed2918634132adcc8645cdc29eca 2024-11-19T15:14:56,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/3786ed2918634132adcc8645cdc29eca, entries=150, sequenceid=228, filesize=11.9 K 2024-11-19T15:14:56,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bd5712f88c144f357fc59afd6d3ab798 in 1254ms, sequenceid=228, compaction requested=true 2024-11-19T15:14:56,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:56,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:56,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:56,832 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:56,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:56,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:56,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:56,833 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:56,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:56,833 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:56,833 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/A is initiating minor compaction (all files) 2024-11-19T15:14:56,833 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:56,834 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/A in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,834 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/B is initiating minor compaction (all files) 2024-11-19T15:14:56,834 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/c2e5c583d85148908ac94f62bc25e391, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b8909d92f1fe4b159735d3c3deeb1a3c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/4be5e23dc1b740b0913299ebaef8455b] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=36.0 K 2024-11-19T15:14:56,834 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/B in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,834 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0cda22b4dc904ab19e3fb86059b96c13, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/b6b71dc088ab43bfb6033b557ef399ca, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/1b432df1bf1143a2912701c375f4a4b5] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=36.0 K 2024-11-19T15:14:56,834 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2e5c583d85148908ac94f62bc25e391, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732029292708 2024-11-19T15:14:56,834 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cda22b4dc904ab19e3fb86059b96c13, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732029292708 2024-11-19T15:14:56,834 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8909d92f1fe4b159735d3c3deeb1a3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732029293384 2024-11-19T15:14:56,835 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4be5e23dc1b740b0913299ebaef8455b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732029295572 2024-11-19T15:14:56,835 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting b6b71dc088ab43bfb6033b557ef399ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732029293384 2024-11-19T15:14:56,835 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b432df1bf1143a2912701c375f4a4b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732029295572 2024-11-19T15:14:56,845 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#B#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:56,845 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#A#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:56,845 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/afa4228ea45b462e843216f89a587d39 is 50, key is test_row_0/B:col10/1732029295572/Put/seqid=0 2024-11-19T15:14:56,845 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/5a7c512ca1334c53944e48c3413ff08c is 50, key is test_row_0/A:col10/1732029295572/Put/seqid=0 2024-11-19T15:14:56,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742115_1291 (size=12697) 2024-11-19T15:14:56,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742116_1292 (size=12697) 2024-11-19T15:14:56,864 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/5a7c512ca1334c53944e48c3413ff08c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/5a7c512ca1334c53944e48c3413ff08c 2024-11-19T15:14:56,869 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/A of bd5712f88c144f357fc59afd6d3ab798 into 5a7c512ca1334c53944e48c3413ff08c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:56,869 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:56,869 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/A, priority=13, startTime=1732029296832; duration=0sec 2024-11-19T15:14:56,869 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:56,869 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:A 2024-11-19T15:14:56,869 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:56,870 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:56,870 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/C is initiating minor compaction (all files) 2024-11-19T15:14:56,870 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/C in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,870 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/44f099879c174d2a8c9d95baa8a58604, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/4e8b6aa3eacb4d91bcc17b50d42e0457, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/3786ed2918634132adcc8645cdc29eca] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=36.0 K 2024-11-19T15:14:56,871 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44f099879c174d2a8c9d95baa8a58604, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732029292708 2024-11-19T15:14:56,871 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e8b6aa3eacb4d91bcc17b50d42e0457, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732029293384 2024-11-19T15:14:56,872 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3786ed2918634132adcc8645cdc29eca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732029295572 2024-11-19T15:14:56,879 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#C#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:56,879 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/dad34d0237564eb8b06271525caee1e7 is 50, key is test_row_0/C:col10/1732029295572/Put/seqid=0 2024-11-19T15:14:56,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742117_1293 (size=12697) 2024-11-19T15:14:56,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T15:14:56,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:56,937 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T15:14:56,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:56,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:56,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:56,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:56,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:56,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:56,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/bb237def9a314cf9aef4506c8ef5585f is 50, key is test_row_0/A:col10/1732029295669/Put/seqid=0 2024-11-19T15:14:56,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742118_1294 (size=12301) 2024-11-19T15:14:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T15:14:57,255 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/afa4228ea45b462e843216f89a587d39 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/afa4228ea45b462e843216f89a587d39 2024-11-19T15:14:57,260 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/B of bd5712f88c144f357fc59afd6d3ab798 into afa4228ea45b462e843216f89a587d39(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:57,260 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:57,260 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/B, priority=13, startTime=1732029296832; duration=0sec 2024-11-19T15:14:57,260 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:57,261 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:B 2024-11-19T15:14:57,289 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/dad34d0237564eb8b06271525caee1e7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/dad34d0237564eb8b06271525caee1e7 2024-11-19T15:14:57,293 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/C of bd5712f88c144f357fc59afd6d3ab798 into dad34d0237564eb8b06271525caee1e7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:57,294 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:57,294 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/C, priority=13, startTime=1732029296833; duration=0sec 2024-11-19T15:14:57,294 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:57,294 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:C 2024-11-19T15:14:57,347 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/bb237def9a314cf9aef4506c8ef5585f 2024-11-19T15:14:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/c3039f3c66314bb097e5f28405b9d69a is 50, key is test_row_0/B:col10/1732029295669/Put/seqid=0 2024-11-19T15:14:57,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742119_1295 (size=12301) 2024-11-19T15:14:57,761 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/c3039f3c66314bb097e5f28405b9d69a 2024-11-19T15:14:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/3846432e85744767933dfce73784d1f7 is 50, key is test_row_0/C:col10/1732029295669/Put/seqid=0 2024-11-19T15:14:57,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742120_1296 (size=12301) 2024-11-19T15:14:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:57,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:14:57,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029357848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:57,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029357851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:57,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:57,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029357964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:57,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:57,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029357964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:58,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T15:14:58,174 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/3846432e85744767933dfce73784d1f7 2024-11-19T15:14:58,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029358174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:58,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029358174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:58,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/bb237def9a314cf9aef4506c8ef5585f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/bb237def9a314cf9aef4506c8ef5585f 2024-11-19T15:14:58,185 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/bb237def9a314cf9aef4506c8ef5585f, entries=150, sequenceid=265, filesize=12.0 K 2024-11-19T15:14:58,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/c3039f3c66314bb097e5f28405b9d69a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/c3039f3c66314bb097e5f28405b9d69a 2024-11-19T15:14:58,193 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/c3039f3c66314bb097e5f28405b9d69a, entries=150, sequenceid=265, filesize=12.0 K 2024-11-19T15:14:58,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/3846432e85744767933dfce73784d1f7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/3846432e85744767933dfce73784d1f7 2024-11-19T15:14:58,199 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/3846432e85744767933dfce73784d1f7, entries=150, sequenceid=265, filesize=12.0 K 2024-11-19T15:14:58,200 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for bd5712f88c144f357fc59afd6d3ab798 in 1264ms, sequenceid=265, compaction requested=false 2024-11-19T15:14:58,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:58,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:58,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-19T15:14:58,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-19T15:14:58,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-19T15:14:58,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1840 sec 2024-11-19T15:14:58,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.1910 sec 2024-11-19T15:14:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:58,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T15:14:58,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:58,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:58,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:58,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:58,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:58,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:14:58,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/bd2d0ea4f62c4961a2d402191deb6eb1 is 50, key is test_row_0/A:col10/1732029298486/Put/seqid=0 2024-11-19T15:14:58,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742121_1297 (size=12301) 2024-11-19T15:14:58,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/bd2d0ea4f62c4961a2d402191deb6eb1 2024-11-19T15:14:58,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/3beace3e657442809c34bcea95bdd1a4 is 50, key is test_row_0/B:col10/1732029298486/Put/seqid=0 2024-11-19T15:14:58,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742122_1298 (size=12301) 2024-11-19T15:14:58,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/3beace3e657442809c34bcea95bdd1a4 2024-11-19T15:14:58,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029358597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:58,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029358599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:58,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/55336742c0c74eadb29064805c766fb6 is 50, key is test_row_0/C:col10/1732029298486/Put/seqid=0 2024-11-19T15:14:58,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742123_1299 (size=12301) 2024-11-19T15:14:58,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/55336742c0c74eadb29064805c766fb6 2024-11-19T15:14:58,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/bd2d0ea4f62c4961a2d402191deb6eb1 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/bd2d0ea4f62c4961a2d402191deb6eb1 2024-11-19T15:14:58,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/bd2d0ea4f62c4961a2d402191deb6eb1, entries=150, sequenceid=279, filesize=12.0 K 2024-11-19T15:14:58,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/3beace3e657442809c34bcea95bdd1a4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/3beace3e657442809c34bcea95bdd1a4 2024-11-19T15:14:58,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/3beace3e657442809c34bcea95bdd1a4, entries=150, sequenceid=279, filesize=12.0 K 2024-11-19T15:14:58,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/55336742c0c74eadb29064805c766fb6 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/55336742c0c74eadb29064805c766fb6 2024-11-19T15:14:58,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/55336742c0c74eadb29064805c766fb6, entries=150, sequenceid=279, filesize=12.0 K 2024-11-19T15:14:58,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for bd5712f88c144f357fc59afd6d3ab798 in 186ms, sequenceid=279, compaction requested=true 2024-11-19T15:14:58,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:58,674 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:58,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:14:58,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:58,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:14:58,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:58,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:14:58,675 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:58,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:58,676 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:58,676 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:58,676 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/B is initiating minor compaction (all files) 2024-11-19T15:14:58,676 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/A is initiating minor compaction (all files) 2024-11-19T15:14:58,676 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/A in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:58,676 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/B in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:58,676 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/afa4228ea45b462e843216f89a587d39, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/c3039f3c66314bb097e5f28405b9d69a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/3beace3e657442809c34bcea95bdd1a4] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=36.4 K 2024-11-19T15:14:58,676 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/5a7c512ca1334c53944e48c3413ff08c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/bb237def9a314cf9aef4506c8ef5585f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/bd2d0ea4f62c4961a2d402191deb6eb1] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=36.4 K 2024-11-19T15:14:58,677 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting afa4228ea45b462e843216f89a587d39, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732029295572 2024-11-19T15:14:58,677 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a7c512ca1334c53944e48c3413ff08c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732029295572 2024-11-19T15:14:58,678 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting c3039f3c66314bb097e5f28405b9d69a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732029295662 2024-11-19T15:14:58,678 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb237def9a314cf9aef4506c8ef5585f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732029295662 2024-11-19T15:14:58,679 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 3beace3e657442809c34bcea95bdd1a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732029297840 2024-11-19T15:14:58,679 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd2d0ea4f62c4961a2d402191deb6eb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732029297840 2024-11-19T15:14:58,699 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:58,699 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#A#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:58,700 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/b104ae8698584f359da9a33a1f8fb75d is 50, key is test_row_0/A:col10/1732029298486/Put/seqid=0 2024-11-19T15:14:58,700 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/301819fd52c54adcbfb124df45477fc3 is 50, key is test_row_0/B:col10/1732029298486/Put/seqid=0 2024-11-19T15:14:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:58,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029298564, max timestamp=1732029298717Num uniques 859; 2024-11-19T15:14:58,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:14:58,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029298564, max timestamp=1732029298717Num uniques 859; 2024-11-19T15:14:58,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029298564, max timestamp=1732029298717Num uniques 859; 2024-11-19T15:14:58,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:14:58,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:14:58,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-11-19T15:14:58,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:58,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:58,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:58,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:58,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:58,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:58,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742125_1301 (size=12949) 2024-11-19T15:14:58,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/092393d21ae04a1da71203041e606032 is 50, key is test_row_0/A:col10/1732029298564/Put/seqid=0 2024-11-19T15:14:58,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742124_1300 (size=12949) 2024-11-19T15:14:58,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742126_1302 (size=17181) 2024-11-19T15:14:58,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732029298763, max timestamp=1732029298798Num uniques 859; 2024-11-19T15:14:58,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:58,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732029298763, max timestamp=1732029298798Num uniques 859; 2024-11-19T15:14:58,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:14:58,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732029298763, max timestamp=1732029298798Num uniques 859; 2024-11-19T15:14:58,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T15:14:58,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732029298852, max timestamp=1732029298874Num uniques 1068; 2024-11-19T15:14:58,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732029298852, max timestamp=1732029298874Num uniques 1068; 2024-11-19T15:14:58,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732029298852, max timestamp=1732029298874Num uniques 1009; 2024-11-19T15:14:58,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-19T15:14:58,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-19T15:14:58,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7688, suffix heap size=132904,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-19T15:14:58,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732029298922, max timestamp=1732029298945Num uniques 1027; 2024-11-19T15:14:58,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732029298922, max timestamp=1732029298945Num uniques 1027; 2024-11-19T15:14:58,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732029298922, max timestamp=1732029298945Num uniques 1027; 2024-11-19T15:14:58,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-19T15:14:58,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-19T15:14:58,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-19T15:14:59,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732029298964, max timestamp=1732029299019Num uniques 1086; 2024-11-19T15:14:59,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732029298964, max timestamp=1732029299019Num uniques 1086; 2024-11-19T15:14:59,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T15:14:59,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T15:14:59,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732029298964, max timestamp=1732029299019Num uniques 1086; 2024-11-19T15:14:59,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T15:14:59,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732029299040, max timestamp=1732029299099Num uniques 1095; 2024-11-19T15:14:59,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T15:14:59,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732029299040, max timestamp=1732029299099Num uniques 1095; 2024-11-19T15:14:59,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T15:14:59,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732029299040, max timestamp=1732029299099Num uniques 1095; 2024-11-19T15:14:59,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T15:14:59,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029359146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:59,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732029299138, max timestamp=1732029299149Num uniques 1104; 2024-11-19T15:14:59,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732029299138, max timestamp=1732029299149Num uniques 1104; 2024-11-19T15:14:59,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-19T15:14:59,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-19T15:14:59,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732029299138, max timestamp=1732029299149Num uniques 1104; 2024-11-19T15:14:59,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-19T15:14:59,161 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/b104ae8698584f359da9a33a1f8fb75d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b104ae8698584f359da9a33a1f8fb75d 2024-11-19T15:14:59,166 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/A of bd5712f88c144f357fc59afd6d3ab798 into b104ae8698584f359da9a33a1f8fb75d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:59,166 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:59,166 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/A, priority=13, startTime=1732029298674; duration=0sec 2024-11-19T15:14:59,166 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:14:59,166 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:A 2024-11-19T15:14:59,166 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:14:59,167 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:14:59,167 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/C is initiating minor compaction (all files) 2024-11-19T15:14:59,167 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/C in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:14:59,168 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/dad34d0237564eb8b06271525caee1e7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/3846432e85744767933dfce73784d1f7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/55336742c0c74eadb29064805c766fb6] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=36.4 K 2024-11-19T15:14:59,168 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting dad34d0237564eb8b06271525caee1e7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732029295572 2024-11-19T15:14:59,168 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3846432e85744767933dfce73784d1f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732029295662 2024-11-19T15:14:59,169 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/301819fd52c54adcbfb124df45477fc3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/301819fd52c54adcbfb124df45477fc3 2024-11-19T15:14:59,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/092393d21ae04a1da71203041e606032 2024-11-19T15:14:59,169 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55336742c0c74eadb29064805c766fb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732029297840 2024-11-19T15:14:59,175 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/B of bd5712f88c144f357fc59afd6d3ab798 into 301819fd52c54adcbfb124df45477fc3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:59,175 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:59,175 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/B, priority=13, startTime=1732029298674; duration=0sec 2024-11-19T15:14:59,175 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:59,175 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:B 2024-11-19T15:14:59,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/0b31d104646f403fa4554f2472e4f2d4 is 50, key is test_row_0/B:col10/1732029298564/Put/seqid=0 2024-11-19T15:14:59,182 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#C#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:14:59,182 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/1480beacae8b49b281d6fd741884b746 is 50, key is test_row_0/C:col10/1732029298486/Put/seqid=0 2024-11-19T15:14:59,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742127_1303 (size=12301) 2024-11-19T15:14:59,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/0b31d104646f403fa4554f2472e4f2d4 2024-11-19T15:14:59,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d82c2f1e37a048679f6d31cbd96219f9 is 50, key is test_row_0/C:col10/1732029298564/Put/seqid=0 2024-11-19T15:14:59,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742128_1304 (size=12949) 2024-11-19T15:14:59,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742129_1305 (size=12301) 2024-11-19T15:14:59,201 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/1480beacae8b49b281d6fd741884b746 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/1480beacae8b49b281d6fd741884b746 2024-11-19T15:14:59,206 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/C of bd5712f88c144f357fc59afd6d3ab798 into 1480beacae8b49b281d6fd741884b746(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:14:59,206 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:59,206 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/C, priority=13, startTime=1732029298675; duration=0sec 2024-11-19T15:14:59,206 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:14:59,206 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:C 2024-11-19T15:14:59,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732029299236, max timestamp=1732029299255Num uniques 1013; 2024-11-19T15:14:59,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-19T15:14:59,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732029299236, max timestamp=1732029299255Num uniques 1013; 2024-11-19T15:14:59,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732029299236, max timestamp=1732029299255Num uniques 1013; 2024-11-19T15:14:59,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-19T15:14:59,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-19T15:14:59,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732029299279, max timestamp=1732029299299Num uniques 1022; 2024-11-19T15:14:59,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732029299279, max timestamp=1732029299299Num uniques 1022; 2024-11-19T15:14:59,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328  suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-19T15:14:59,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328  suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-19T15:14:59,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732029299279, max timestamp=1732029299299Num uniques 1022; 2024-11-19T15:14:59,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328  suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-19T15:14:59,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732029299342, max timestamp=1732029299362Num uniques 1081; 2024-11-19T15:14:59,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064  suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-19T15:14:59,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732029299342, max timestamp=1732029299365Num uniques 1081; 2024-11-19T15:14:59,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064  suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-19T15:14:59,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732029299342, max timestamp=1732029299365Num uniques 1081; 2024-11-19T15:14:59,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064  suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-19T15:14:59,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732029299407, max timestamp=1732029299420Num uniques 1040; 2024-11-19T15:14:59,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000  suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-19T15:14:59,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732029299407, max timestamp=1732029299420Num uniques 1040; 2024-11-19T15:14:59,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000  suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-19T15:14:59,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732029299407, max timestamp=1732029299420Num uniques 1040; 2024-11-19T15:14:59,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000  suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-19T15:14:59,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732029299466, max timestamp=1732029299475Num uniques 1049; 2024-11-19T15:14:59,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936  suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-19T15:14:59,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732029299466, max timestamp=1732029299475Num uniques 1049; 2024-11-19T15:14:59,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936  suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-19T15:14:59,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732029299466, max timestamp=1732029299475Num uniques 1049; 2024-11-19T15:14:59,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936  suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-19T15:14:59,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1732029299516, max timestamp=1732029299530Num uniques 1058; 2024-11-19T15:14:59,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872  suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-11-19T15:14:59,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1732029299516, max timestamp=1732029299530Num uniques 1058; 2024-11-19T15:14:59,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872  suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-11-19T15:14:59,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1732029299516, max timestamp=1732029299530Num uniques 1058; 2024-11-19T15:14:59,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872  suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-11-19T15:14:59,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d82c2f1e37a048679f6d31cbd96219f9 2024-11-19T15:14:59,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732029299581, max timestamp=1732029299598Num uniques 1067; 2024-11-19T15:14:59,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808  suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-19T15:14:59,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732029299581, max timestamp=1732029299598Num uniques 1067; 2024-11-19T15:14:59,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808  suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-19T15:14:59,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732029299581, max timestamp=1732029299598Num uniques 1067; 2024-11-19T15:14:59,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808  suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-19T15:14:59,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/092393d21ae04a1da71203041e606032 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/092393d21ae04a1da71203041e606032 2024-11-19T15:14:59,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/092393d21ae04a1da71203041e606032, entries=250, sequenceid=301, filesize=16.8 K 2024-11-19T15:14:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/0b31d104646f403fa4554f2472e4f2d4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0b31d104646f403fa4554f2472e4f2d4 2024-11-19T15:14:59,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0b31d104646f403fa4554f2472e4f2d4, entries=150, sequenceid=301, filesize=12.0 K 2024-11-19T15:14:59,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d82c2f1e37a048679f6d31cbd96219f9 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d82c2f1e37a048679f6d31cbd96219f9 2024-11-19T15:14:59,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d82c2f1e37a048679f6d31cbd96219f9, entries=150, sequenceid=301, filesize=12.0 K 2024-11-19T15:14:59,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=114.05 KB/116790 for bd5712f88c144f357fc59afd6d3ab798 in 912ms, sequenceid=301, compaction requested=false 2024-11-19T15:14:59,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:14:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:14:59,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=120.76 KB heapSize=339.70 KB 2024-11-19T15:14:59,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:14:59,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:59,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:14:59,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:59,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:14:59,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:14:59,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e21b25cb837f41d08c7e453b0851a705 is 50, key is test_row_0/A:col10/1732029299625/Put/seqid=0 2024-11-19T15:14:59,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742130_1306 (size=12301) 2024-11-19T15:14:59,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029359657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:59,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:59,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029359662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:59,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029359774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:59,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029359774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:59,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029359984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:14:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:14:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029359984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:14:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:14:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:00,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e21b25cb837f41d08c7e453b0851a705 2024-11-19T15:15:00,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/4be4cb0ed4934ea0a168a04a2050a9ad is 50, key is test_row_0/B:col10/1732029299625/Put/seqid=0 2024-11-19T15:15:00,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742131_1307 (size=12301) 2024-11-19T15:15:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T15:15:00,185 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:00,185 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-19T15:15:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T15:15:00,187 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:00,188 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:00,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T15:15:00,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029360294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:00,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029360294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:00,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T15:15:00,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:00,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:00,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:00,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:00,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:00,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/4be4cb0ed4934ea0a168a04a2050a9ad 2024-11-19T15:15:00,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/28baff02eeac4cbbb94704f10d74c1e3 is 50, key is test_row_0/C:col10/1732029299625/Put/seqid=0 2024-11-19T15:15:00,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742132_1308 (size=12301) 2024-11-19T15:15:00,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T15:15:00,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:00,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:00,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:00,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:00,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T15:15:00,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T15:15:00,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:00,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:00,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:00,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:00,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:00,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T15:15:00,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:00,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:00,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:00,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:00,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:00,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029360804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:00,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029360804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T15:15:00,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/28baff02eeac4cbbb94704f10d74c1e3 2024-11-19T15:15:00,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e21b25cb837f41d08c7e453b0851a705 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e21b25cb837f41d08c7e453b0851a705 2024-11-19T15:15:00,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e21b25cb837f41d08c7e453b0851a705, entries=150, sequenceid=544, filesize=12.0 K 2024-11-19T15:15:00,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/4be4cb0ed4934ea0a168a04a2050a9ad as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4be4cb0ed4934ea0a168a04a2050a9ad 2024-11-19T15:15:00,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4be4cb0ed4934ea0a168a04a2050a9ad, entries=150, sequenceid=544, filesize=12.0 K 2024-11-19T15:15:00,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/28baff02eeac4cbbb94704f10d74c1e3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/28baff02eeac4cbbb94704f10d74c1e3 2024-11-19T15:15:00,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/28baff02eeac4cbbb94704f10d74c1e3, entries=150, sequenceid=544, filesize=12.0 K 2024-11-19T15:15:00,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~339.66 KB/347808, currentSize=60.38 KB/61830 for bd5712f88c144f357fc59afd6d3ab798 in 1265ms, sequenceid=544, compaction requested=true 2024-11-19T15:15:00,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:00,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:00,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:00,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:00,903 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:00,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:00,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:00,903 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:00,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:00,905 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:00,905 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/B is initiating minor compaction (all files) 2024-11-19T15:15:00,905 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/B in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:00,906 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/301819fd52c54adcbfb124df45477fc3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0b31d104646f403fa4554f2472e4f2d4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4be4cb0ed4934ea0a168a04a2050a9ad] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=36.7 K 2024-11-19T15:15:00,906 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:00,906 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 301819fd52c54adcbfb124df45477fc3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732029297840 2024-11-19T15:15:00,906 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/A is initiating minor compaction (all files) 2024-11-19T15:15:00,906 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/A in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:00,906 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b104ae8698584f359da9a33a1f8fb75d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/092393d21ae04a1da71203041e606032, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e21b25cb837f41d08c7e453b0851a705] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=41.4 K 2024-11-19T15:15:00,907 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b31d104646f403fa4554f2472e4f2d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732029298564 2024-11-19T15:15:00,907 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting b104ae8698584f359da9a33a1f8fb75d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732029297840 2024-11-19T15:15:00,907 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 4be4cb0ed4934ea0a168a04a2050a9ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1732029299625 2024-11-19T15:15:00,907 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 092393d21ae04a1da71203041e606032, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732029298564 2024-11-19T15:15:00,907 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e21b25cb837f41d08c7e453b0851a705, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1732029299625 2024-11-19T15:15:00,917 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#B#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:00,917 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/6ab2db1a46d14da793f1d3acc4a3afc2 is 50, key is test_row_0/B:col10/1732029299625/Put/seqid=0 2024-11-19T15:15:00,919 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#A#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:00,919 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/34cd3b0bb7254064b786b8aa97783063 is 50, key is test_row_0/A:col10/1732029299625/Put/seqid=0 2024-11-19T15:15:00,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742133_1309 (size=13051) 2024-11-19T15:15:00,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742134_1310 (size=13051) 2024-11-19T15:15:00,933 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/6ab2db1a46d14da793f1d3acc4a3afc2 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/6ab2db1a46d14da793f1d3acc4a3afc2 2024-11-19T15:15:00,935 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/34cd3b0bb7254064b786b8aa97783063 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/34cd3b0bb7254064b786b8aa97783063 2024-11-19T15:15:00,939 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/B of bd5712f88c144f357fc59afd6d3ab798 into 6ab2db1a46d14da793f1d3acc4a3afc2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:00,939 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:00,939 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/B, priority=13, startTime=1732029300903; duration=0sec 2024-11-19T15:15:00,939 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:00,939 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:B 2024-11-19T15:15:00,939 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:00,940 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/A of bd5712f88c144f357fc59afd6d3ab798 into 34cd3b0bb7254064b786b8aa97783063(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:00,940 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:00,940 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/A, priority=13, startTime=1732029300903; duration=0sec 2024-11-19T15:15:00,940 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:00,940 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:A 2024-11-19T15:15:00,940 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:00,940 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/C is initiating minor compaction (all files) 2024-11-19T15:15:00,940 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/C in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:00,941 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/1480beacae8b49b281d6fd741884b746, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d82c2f1e37a048679f6d31cbd96219f9, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/28baff02eeac4cbbb94704f10d74c1e3] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=36.7 K 2024-11-19T15:15:00,941 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 1480beacae8b49b281d6fd741884b746, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732029297840 2024-11-19T15:15:00,941 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d82c2f1e37a048679f6d31cbd96219f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732029298564 2024-11-19T15:15:00,941 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 28baff02eeac4cbbb94704f10d74c1e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1732029299625 2024-11-19T15:15:00,948 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#C#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:00,949 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/14a4c2bbecec43dfbebbecdbc23a1936 is 50, key is test_row_0/C:col10/1732029299625/Put/seqid=0 2024-11-19T15:15:00,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742135_1311 (size=13051) 2024-11-19T15:15:00,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T15:15:00,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:00,955 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T15:15:00,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:15:00,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:00,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:15:00,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:00,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:15:00,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:00,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/fcc5c5137a3548ae9b61bc5a06ddcc4c is 50, key is test_row_0/A:col10/1732029299656/Put/seqid=0 2024-11-19T15:15:00,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742136_1312 (size=12301) 2024-11-19T15:15:00,966 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/fcc5c5137a3548ae9b61bc5a06ddcc4c 2024-11-19T15:15:00,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/d5988d4dd43045749510a7900c663941 is 50, key is test_row_0/B:col10/1732029299656/Put/seqid=0 2024-11-19T15:15:00,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742137_1313 (size=12301) 2024-11-19T15:15:00,982 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/d5988d4dd43045749510a7900c663941 2024-11-19T15:15:00,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d47996751412460db8ac05e45af23be1 is 50, key is test_row_0/C:col10/1732029299656/Put/seqid=0 2024-11-19T15:15:00,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742138_1314 (size=12301) 2024-11-19T15:15:01,001 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d47996751412460db8ac05e45af23be1 2024-11-19T15:15:01,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/fcc5c5137a3548ae9b61bc5a06ddcc4c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/fcc5c5137a3548ae9b61bc5a06ddcc4c 2024-11-19T15:15:01,016 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/fcc5c5137a3548ae9b61bc5a06ddcc4c, entries=150, sequenceid=558, filesize=12.0 K 2024-11-19T15:15:01,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/d5988d4dd43045749510a7900c663941 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/d5988d4dd43045749510a7900c663941 2024-11-19T15:15:01,023 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/d5988d4dd43045749510a7900c663941, entries=150, sequenceid=558, filesize=12.0 K 2024-11-19T15:15:01,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d47996751412460db8ac05e45af23be1 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d47996751412460db8ac05e45af23be1 2024-11-19T15:15:01,030 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d47996751412460db8ac05e45af23be1, entries=150, sequenceid=558, filesize=12.0 K 2024-11-19T15:15:01,031 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for bd5712f88c144f357fc59afd6d3ab798 in 76ms, sequenceid=558, compaction requested=false 2024-11-19T15:15:01,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:01,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:01,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-19T15:15:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-19T15:15:01,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-19T15:15:01,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 844 msec 2024-11-19T15:15:01,036 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 850 msec 2024-11-19T15:15:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T15:15:01,325 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:01,325 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-19T15:15:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T15:15:01,327 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:01,328 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:01,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:01,358 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/14a4c2bbecec43dfbebbecdbc23a1936 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/14a4c2bbecec43dfbebbecdbc23a1936 2024-11-19T15:15:01,370 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/C of bd5712f88c144f357fc59afd6d3ab798 into 14a4c2bbecec43dfbebbecdbc23a1936(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:01,370 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:01,370 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/C, priority=13, startTime=1732029300903; duration=0sec 2024-11-19T15:15:01,370 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:01,370 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:C 2024-11-19T15:15:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T15:15:01,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T15:15:01,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:01,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:01,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:01,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-19T15:15:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-19T15:15:01,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-19T15:15:01,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-19T15:15:01,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 161 msec 2024-11-19T15:15:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T15:15:01,644 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:01,645 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-19T15:15:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T15:15:01,647 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:01,648 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:01,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T15:15:01,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T15:15:01,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:01,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:01,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:01,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-19T15:15:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-19T15:15:01,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-19T15:15:01,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-19T15:15:01,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 161 msec 2024-11-19T15:15:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:15:01,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:15:01,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:15:01,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:01,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:15:01,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:01,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:15:01,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:01,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e1f87d5f530a40c8ad91c4b30e1d1aa8 is 50, key is test_row_0/A:col10/1732029301827/Put/seqid=0 2024-11-19T15:15:01,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742139_1315 (size=9857) 2024-11-19T15:15:01,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029301887, max timestamp=1732029301902Num uniques 859; 2024-11-19T15:15:01,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:15:01,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029301887, max timestamp=1732029301902Num uniques 859; 2024-11-19T15:15:01,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029301887, max timestamp=1732029301902Num uniques 859; 2024-11-19T15:15:01,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:15:01,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:15:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T15:15:01,964 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:01,965 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-19T15:15:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T15:15:01,966 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:01,967 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:01,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:01,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029361985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:01,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029361986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T15:15:02,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029362094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:02,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029362094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:02,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:02,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T15:15:02,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:02,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:02,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:02,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:02,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:02,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e1f87d5f530a40c8ad91c4b30e1d1aa8 2024-11-19T15:15:02,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/fae504a1a6054195a7556b759b219b70 is 50, key is test_row_0/B:col10/1732029301827/Put/seqid=0 2024-11-19T15:15:02,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742140_1316 (size=9857) 2024-11-19T15:15:02,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T15:15:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:02,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:02,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T15:15:02,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029362304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:02,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:02,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:02,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029362304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:02,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:02,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T15:15:02,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:02,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:02,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:02,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:02,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:02,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T15:15:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:02,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T15:15:02,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029362614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:02,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029362615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:02,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/fae504a1a6054195a7556b759b219b70 2024-11-19T15:15:02,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d8dff4fd0e984345bdefbd251a35903a is 50, key is test_row_0/C:col10/1732029301827/Put/seqid=0 2024-11-19T15:15:02,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742141_1317 (size=9857) 2024-11-19T15:15:02,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d8dff4fd0e984345bdefbd251a35903a 2024-11-19T15:15:02,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e1f87d5f530a40c8ad91c4b30e1d1aa8 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e1f87d5f530a40c8ad91c4b30e1d1aa8 2024-11-19T15:15:02,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e1f87d5f530a40c8ad91c4b30e1d1aa8, entries=100, sequenceid=571, filesize=9.6 K 2024-11-19T15:15:02,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/fae504a1a6054195a7556b759b219b70 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/fae504a1a6054195a7556b759b219b70 2024-11-19T15:15:02,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/fae504a1a6054195a7556b759b219b70, entries=100, sequenceid=571, filesize=9.6 K 2024-11-19T15:15:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/d8dff4fd0e984345bdefbd251a35903a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d8dff4fd0e984345bdefbd251a35903a 2024-11-19T15:15:02,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d8dff4fd0e984345bdefbd251a35903a, entries=100, sequenceid=571, filesize=9.6 K 2024-11-19T15:15:02,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bd5712f88c144f357fc59afd6d3ab798 in 857ms, sequenceid=571, compaction requested=true 2024-11-19T15:15:02,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:02,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:02,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:02,686 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:02,686 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:02,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:02,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:02,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:02,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:02,687 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:02,687 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:02,687 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/A is initiating minor compaction (all files) 2024-11-19T15:15:02,687 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/B is initiating minor compaction (all files) 2024-11-19T15:15:02,687 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/A in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:02,687 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/B in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:02,687 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/34cd3b0bb7254064b786b8aa97783063, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/fcc5c5137a3548ae9b61bc5a06ddcc4c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e1f87d5f530a40c8ad91c4b30e1d1aa8] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=34.4 K 2024-11-19T15:15:02,687 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/6ab2db1a46d14da793f1d3acc4a3afc2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/d5988d4dd43045749510a7900c663941, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/fae504a1a6054195a7556b759b219b70] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=34.4 K 2024-11-19T15:15:02,688 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ab2db1a46d14da793f1d3acc4a3afc2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1732029299625 2024-11-19T15:15:02,688 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34cd3b0bb7254064b786b8aa97783063, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1732029299625 2024-11-19T15:15:02,688 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcc5c5137a3548ae9b61bc5a06ddcc4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1732029299645 2024-11-19T15:15:02,688 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d5988d4dd43045749510a7900c663941, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1732029299645 2024-11-19T15:15:02,688 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1f87d5f530a40c8ad91c4b30e1d1aa8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732029301822 2024-11-19T15:15:02,688 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting fae504a1a6054195a7556b759b219b70, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732029301822 2024-11-19T15:15:02,696 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#A#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:02,696 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#B#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:02,697 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/75dd0976a3554a57bb8ecfb0753a3c5f is 50, key is test_row_0/B:col10/1732029301827/Put/seqid=0 2024-11-19T15:15:02,697 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e36b6f1eb6dd4791bea42e0f4b0ab333 is 50, key is test_row_0/A:col10/1732029301827/Put/seqid=0 2024-11-19T15:15:02,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742143_1319 (size=13153) 2024-11-19T15:15:02,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742142_1318 (size=13153) 2024-11-19T15:15:02,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T15:15:02,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:02,734 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T15:15:02,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:15:02,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:02,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:15:02,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:02,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:15:02,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:02,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/37cdd778ddeb4abea833cbf25d4966c4 is 50, key is test_row_0/A:col10/1732029301971/Put/seqid=0 2024-11-19T15:15:02,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742144_1320 (size=12301) 2024-11-19T15:15:02,744 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/37cdd778ddeb4abea833cbf25d4966c4 2024-11-19T15:15:02,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/1d2376ffaa784ddda054fb2ea41cb001 is 50, key is test_row_0/B:col10/1732029301971/Put/seqid=0 2024-11-19T15:15:02,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742145_1321 (size=12301) 2024-11-19T15:15:02,769 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/1d2376ffaa784ddda054fb2ea41cb001 2024-11-19T15:15:02,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/25fd548d3638463b9f22ebd61bbe8db5 is 50, key is test_row_0/C:col10/1732029301971/Put/seqid=0 2024-11-19T15:15:02,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742146_1322 (size=12301) 2024-11-19T15:15:02,803 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/25fd548d3638463b9f22ebd61bbe8db5 2024-11-19T15:15:02,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/37cdd778ddeb4abea833cbf25d4966c4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/37cdd778ddeb4abea833cbf25d4966c4 2024-11-19T15:15:02,822 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/37cdd778ddeb4abea833cbf25d4966c4, entries=150, sequenceid=608, filesize=12.0 K 2024-11-19T15:15:02,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/1d2376ffaa784ddda054fb2ea41cb001 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/1d2376ffaa784ddda054fb2ea41cb001 2024-11-19T15:15:02,828 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/1d2376ffaa784ddda054fb2ea41cb001, entries=150, sequenceid=608, filesize=12.0 K 2024-11-19T15:15:02,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/25fd548d3638463b9f22ebd61bbe8db5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/25fd548d3638463b9f22ebd61bbe8db5 2024-11-19T15:15:02,835 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/25fd548d3638463b9f22ebd61bbe8db5, entries=150, sequenceid=608, filesize=12.0 K 2024-11-19T15:15:02,839 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for bd5712f88c144f357fc59afd6d3ab798 in 106ms, sequenceid=608, compaction requested=true 2024-11-19T15:15:02,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:02,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:02,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-19T15:15:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-19T15:15:02,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-19T15:15:02,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 873 msec 2024-11-19T15:15:02,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 879 msec 2024-11-19T15:15:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T15:15:03,104 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:03,105 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-19T15:15:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T15:15:03,107 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:03,108 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:03,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:03,117 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/e36b6f1eb6dd4791bea42e0f4b0ab333 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e36b6f1eb6dd4791bea42e0f4b0ab333 2024-11-19T15:15:03,118 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/75dd0976a3554a57bb8ecfb0753a3c5f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/75dd0976a3554a57bb8ecfb0753a3c5f 2024-11-19T15:15:03,133 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/B of bd5712f88c144f357fc59afd6d3ab798 into 75dd0976a3554a57bb8ecfb0753a3c5f(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:03,133 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/A of bd5712f88c144f357fc59afd6d3ab798 into e36b6f1eb6dd4791bea42e0f4b0ab333(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:03,133 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:03,133 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:03,133 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/A, priority=13, startTime=1732029302686; duration=0sec 2024-11-19T15:15:03,133 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/B, priority=13, startTime=1732029302686; duration=0sec 2024-11-19T15:15:03,134 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:03,134 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:A 2024-11-19T15:15:03,134 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:03,134 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:B 2024-11-19T15:15:03,134 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:15:03,136 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:15:03,137 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/C is initiating minor compaction (all files) 2024-11-19T15:15:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:15:03,137 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/C in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:03,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:15:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:15:03,137 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/14a4c2bbecec43dfbebbecdbc23a1936, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d47996751412460db8ac05e45af23be1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d8dff4fd0e984345bdefbd251a35903a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/25fd548d3638463b9f22ebd61bbe8db5] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=46.4 K 2024-11-19T15:15:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:15:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:15:03,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:03,138 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14a4c2bbecec43dfbebbecdbc23a1936, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1732029299625 2024-11-19T15:15:03,138 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting d47996751412460db8ac05e45af23be1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1732029299645 2024-11-19T15:15:03,139 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8dff4fd0e984345bdefbd251a35903a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732029301822 2024-11-19T15:15:03,139 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25fd548d3638463b9f22ebd61bbe8db5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1732029301971 2024-11-19T15:15:03,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/142057b5c2bb4672aeec53f0295a89eb is 50, key is test_row_1/A:col10/1732029303134/Put/seqid=0 2024-11-19T15:15:03,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742147_1323 (size=14737) 2024-11-19T15:15:03,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/142057b5c2bb4672aeec53f0295a89eb 2024-11-19T15:15:03,163 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#C#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:03,165 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/e99ca1d7bb704068b197e25a750f8f23 is 50, key is test_row_0/C:col10/1732029301971/Put/seqid=0 2024-11-19T15:15:03,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/7a575dd8dc9d4c67ab05b0997037624f is 50, key is test_row_1/B:col10/1732029303134/Put/seqid=0 2024-11-19T15:15:03,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742148_1324 (size=13187) 2024-11-19T15:15:03,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742149_1325 (size=9857) 2024-11-19T15:15:03,183 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/e99ca1d7bb704068b197e25a750f8f23 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/e99ca1d7bb704068b197e25a750f8f23 2024-11-19T15:15:03,187 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/C of bd5712f88c144f357fc59afd6d3ab798 into e99ca1d7bb704068b197e25a750f8f23(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:03,187 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:03,187 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/C, priority=12, startTime=1732029302686; duration=0sec 2024-11-19T15:15:03,187 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:03,187 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:C 2024-11-19T15:15:03,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029303164, max timestamp=1732029303195Num uniques 859; 2024-11-19T15:15:03,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:15:03,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029303164, max timestamp=1732029303195Num uniques 859; 2024-11-19T15:15:03,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:15:03,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029303164, max timestamp=1732029303195Num uniques 859; 2024-11-19T15:15:03,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:15:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T15:15:03,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029363237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:03,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029363240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:03,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T15:15:03,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:03,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:03,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:03,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 328 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029363344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:03,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029363354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:03,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T15:15:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:03,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T15:15:03,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 329 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029363554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:03,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T15:15:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:03,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029363564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:03,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/7a575dd8dc9d4c67ab05b0997037624f 2024-11-19T15:15:03,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/035a29970f4f48ffab2f4c8d340f1221 is 50, key is test_row_1/C:col10/1732029303134/Put/seqid=0 2024-11-19T15:15:03,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742150_1326 (size=9857) 2024-11-19T15:15:03,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T15:15:03,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:03,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:03,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:03,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T15:15:03,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 330 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029363864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:03,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T15:15:03,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:03,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:03,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:03,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:03,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029363874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:03,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/035a29970f4f48ffab2f4c8d340f1221 2024-11-19T15:15:03,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/142057b5c2bb4672aeec53f0295a89eb as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/142057b5c2bb4672aeec53f0295a89eb 2024-11-19T15:15:03,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/142057b5c2bb4672aeec53f0295a89eb, entries=200, sequenceid=620, filesize=14.4 K 2024-11-19T15:15:04,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/7a575dd8dc9d4c67ab05b0997037624f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/7a575dd8dc9d4c67ab05b0997037624f 2024-11-19T15:15:04,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/7a575dd8dc9d4c67ab05b0997037624f, entries=100, sequenceid=620, filesize=9.6 K 2024-11-19T15:15:04,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/035a29970f4f48ffab2f4c8d340f1221 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/035a29970f4f48ffab2f4c8d340f1221 2024-11-19T15:15:04,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/035a29970f4f48ffab2f4c8d340f1221, entries=100, sequenceid=620, filesize=9.6 K 2024-11-19T15:15:04,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bd5712f88c144f357fc59afd6d3ab798 in 871ms, sequenceid=620, compaction requested=true 2024-11-19T15:15:04,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:04,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:04,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:04,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:04,009 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:04,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:04,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:04,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:04,009 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:04,010 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40191 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:04,010 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/A is initiating minor compaction (all files) 2024-11-19T15:15:04,010 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/A in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:04,010 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:04,010 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/B is initiating minor compaction (all files) 2024-11-19T15:15:04,010 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e36b6f1eb6dd4791bea42e0f4b0ab333, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/37cdd778ddeb4abea833cbf25d4966c4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/142057b5c2bb4672aeec53f0295a89eb] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=39.2 K 2024-11-19T15:15:04,010 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/B in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:04,010 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/75dd0976a3554a57bb8ecfb0753a3c5f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/1d2376ffaa784ddda054fb2ea41cb001, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/7a575dd8dc9d4c67ab05b0997037624f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=34.5 K 2024-11-19T15:15:04,011 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 75dd0976a3554a57bb8ecfb0753a3c5f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732029299645 2024-11-19T15:15:04,011 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e36b6f1eb6dd4791bea42e0f4b0ab333, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732029299645 2024-11-19T15:15:04,011 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d2376ffaa784ddda054fb2ea41cb001, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1732029301971 2024-11-19T15:15:04,011 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37cdd778ddeb4abea833cbf25d4966c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1732029301971 2024-11-19T15:15:04,011 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a575dd8dc9d4c67ab05b0997037624f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1732029303134 2024-11-19T15:15:04,011 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 142057b5c2bb4672aeec53f0295a89eb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1732029303130 2024-11-19T15:15:04,019 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:04,019 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/9da3ed874e5e440f8eebac8d73865ae8 is 50, key is test_row_0/B:col10/1732029301971/Put/seqid=0 2024-11-19T15:15:04,019 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#A#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:04,020 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/59ab0d4b063e40d4b5244f65996c31af is 50, key is test_row_0/A:col10/1732029301971/Put/seqid=0 2024-11-19T15:15:04,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T15:15:04,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:04,028 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T15:15:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:15:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:15:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:15:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:04,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742152_1328 (size=13255) 2024-11-19T15:15:04,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742151_1327 (size=13255) 2024-11-19T15:15:04,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/67ae24b4747943e1a3fc8fc70416e947 is 50, key is test_row_0/A:col10/1732029303238/Put/seqid=0 2024-11-19T15:15:04,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742153_1329 (size=12301) 2024-11-19T15:15:04,048 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/59ab0d4b063e40d4b5244f65996c31af as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/59ab0d4b063e40d4b5244f65996c31af 2024-11-19T15:15:04,048 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/67ae24b4747943e1a3fc8fc70416e947 2024-11-19T15:15:04,049 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/9da3ed874e5e440f8eebac8d73865ae8 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/9da3ed874e5e440f8eebac8d73865ae8 2024-11-19T15:15:04,055 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/A of bd5712f88c144f357fc59afd6d3ab798 into 59ab0d4b063e40d4b5244f65996c31af(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:04,055 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:04,055 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/A, priority=13, startTime=1732029304009; duration=0sec 2024-11-19T15:15:04,055 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:04,055 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:A 2024-11-19T15:15:04,055 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/B of bd5712f88c144f357fc59afd6d3ab798 into 9da3ed874e5e440f8eebac8d73865ae8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:04,055 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:04,055 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T15:15:04,055 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/B, priority=13, startTime=1732029304009; duration=0sec 2024-11-19T15:15:04,056 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:04,056 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:B 2024-11-19T15:15:04,056 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T15:15:04,056 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T15:15:04,057 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. because compaction request was cancelled 2024-11-19T15:15:04,057 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:C 2024-11-19T15:15:04,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/f00f45481d594e2cb95251e8ba6c3af3 is 50, key is test_row_0/B:col10/1732029303238/Put/seqid=0 2024-11-19T15:15:04,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742154_1330 (size=12301) 2024-11-19T15:15:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T15:15:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:15:04,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:04,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029364400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:04,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029364401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:04,465 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/f00f45481d594e2cb95251e8ba6c3af3 2024-11-19T15:15:04,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/ddc6a67768ab4a8caac5eb9e5593719f is 50, key is test_row_0/C:col10/1732029303238/Put/seqid=0 2024-11-19T15:15:04,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742155_1331 (size=12301) 2024-11-19T15:15:04,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:04,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 337 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029364514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:04,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:04,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029364514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:04,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:04,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:04,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37386 deadline: 1732029364624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:04,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37312 deadline: 1732029364624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:04,628 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in bd5712f88c144f357fc59afd6d3ab798 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:04,628 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in bd5712f88c144f357fc59afd6d3ab798 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:04,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37398 deadline: 1732029364654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:04,659 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in bd5712f88c144f357fc59afd6d3ab798 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18319 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:04,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 338 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029364724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:04,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:04,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029364725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:04,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:04,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:04,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:04,878 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/ddc6a67768ab4a8caac5eb9e5593719f 2024-11-19T15:15:04,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/67ae24b4747943e1a3fc8fc70416e947 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/67ae24b4747943e1a3fc8fc70416e947 2024-11-19T15:15:04,886 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/67ae24b4747943e1a3fc8fc70416e947, entries=150, sequenceid=658, filesize=12.0 K 2024-11-19T15:15:04,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/f00f45481d594e2cb95251e8ba6c3af3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/f00f45481d594e2cb95251e8ba6c3af3 2024-11-19T15:15:04,890 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/f00f45481d594e2cb95251e8ba6c3af3, entries=150, sequenceid=658, filesize=12.0 K 2024-11-19T15:15:04,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/ddc6a67768ab4a8caac5eb9e5593719f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/ddc6a67768ab4a8caac5eb9e5593719f 2024-11-19T15:15:04,895 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/ddc6a67768ab4a8caac5eb9e5593719f, entries=150, sequenceid=658, filesize=12.0 K 2024-11-19T15:15:04,895 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for bd5712f88c144f357fc59afd6d3ab798 in 867ms, sequenceid=658, compaction requested=true 2024-11-19T15:15:04,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:04,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:04,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-19T15:15:04,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-19T15:15:04,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-19T15:15:04,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7890 sec 2024-11-19T15:15:04,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.7960 sec 2024-11-19T15:15:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:15:05,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T15:15:05,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:15:05,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:05,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:15:05,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:05,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:15:05,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:05,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/0f416444b99f489f89564be7868437a4 is 50, key is test_row_0/A:col10/1732029304399/Put/seqid=0 2024-11-19T15:15:05,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742156_1332 (size=17181) 2024-11-19T15:15:05,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/0f416444b99f489f89564be7868437a4 2024-11-19T15:15:05,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/98e8cd11e2354050acfa9f4ca3652376 is 50, key is test_row_0/B:col10/1732029304399/Put/seqid=0 2024-11-19T15:15:05,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742157_1333 (size=12301) 2024-11-19T15:15:05,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:05,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 349 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029365085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029365085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:05,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:05,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 350 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029365194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:05,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029365194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T15:15:05,254 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:05,255 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-19T15:15:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T15:15:05,257 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:05,258 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:05,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T15:15:05,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:05,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029365405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029365404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:05,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:05,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:05,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T15:15:05,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:05,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:05,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:05,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:05,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:05,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/98e8cd11e2354050acfa9f4ca3652376 2024-11-19T15:15:05,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/671719e0baa34898b65a814227a66466 is 50, key is test_row_0/C:col10/1732029304399/Put/seqid=0 2024-11-19T15:15:05,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742158_1334 (size=12301) 2024-11-19T15:15:05,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T15:15:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:05,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:05,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:05,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:05,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T15:15:05,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T15:15:05,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:05,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:05,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:05,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:05,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:05,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37378 deadline: 1732029365714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:05,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 352 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37336 deadline: 1732029365714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:05,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:05,779 INFO [Thread-1139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1139. 2024-11-19T15:15:05,779 DEBUG [Thread-1139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:05,779 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:05,779 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:05,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:05,780 INFO [Thread-1141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1141. 2024-11-19T15:15:05,780 DEBUG [Thread-1141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:05,780 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:05,780 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-11-19T15:15:05,780 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:05,780 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:05,780 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:05,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:05,780 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:05,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:05,784 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-11-19T15:15:05,784 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:05,784 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:05,784 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:05,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:05,784 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-19T15:15:05,784 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:05,784 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:05,784 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:05,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:05,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T15:15:05,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:05,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:05,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:05,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:05,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:05,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/671719e0baa34898b65a814227a66466 2024-11-19T15:15:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T15:15:05,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/0f416444b99f489f89564be7868437a4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/0f416444b99f489f89564be7868437a4 2024-11-19T15:15:05,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/0f416444b99f489f89564be7868437a4, entries=250, sequenceid=673, filesize=16.8 K 2024-11-19T15:15:05,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/98e8cd11e2354050acfa9f4ca3652376 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/98e8cd11e2354050acfa9f4ca3652376 2024-11-19T15:15:05,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/98e8cd11e2354050acfa9f4ca3652376, entries=150, sequenceid=673, filesize=12.0 K 2024-11-19T15:15:05,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/671719e0baa34898b65a814227a66466 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/671719e0baa34898b65a814227a66466 2024-11-19T15:15:05,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/671719e0baa34898b65a814227a66466, entries=150, sequenceid=673, filesize=12.0 K 2024-11-19T15:15:05,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for bd5712f88c144f357fc59afd6d3ab798 in 862ms, sequenceid=673, compaction requested=true 2024-11-19T15:15:05,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:05,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:05,898 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:05,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:05,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:05,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:05,898 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:05,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bd5712f88c144f357fc59afd6d3ab798:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:05,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:05,899 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:05,899 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/A is initiating minor compaction (all files) 2024-11-19T15:15:05,899 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:05,899 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/B is initiating minor compaction (all files) 2024-11-19T15:15:05,899 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/A in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:05,899 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/B in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:05,899 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/9da3ed874e5e440f8eebac8d73865ae8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/f00f45481d594e2cb95251e8ba6c3af3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/98e8cd11e2354050acfa9f4ca3652376] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=37.0 K 2024-11-19T15:15:05,899 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/59ab0d4b063e40d4b5244f65996c31af, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/67ae24b4747943e1a3fc8fc70416e947, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/0f416444b99f489f89564be7868437a4] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=41.7 K 2024-11-19T15:15:05,899 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59ab0d4b063e40d4b5244f65996c31af, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1732029301971 2024-11-19T15:15:05,899 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 9da3ed874e5e440f8eebac8d73865ae8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1732029301971 2024-11-19T15:15:05,900 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67ae24b4747943e1a3fc8fc70416e947, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1732029303211 2024-11-19T15:15:05,900 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting f00f45481d594e2cb95251e8ba6c3af3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1732029303211 2024-11-19T15:15:05,900 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f416444b99f489f89564be7868437a4, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=673, earliestPutTs=1732029304381 2024-11-19T15:15:05,900 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 98e8cd11e2354050acfa9f4ca3652376, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=673, earliestPutTs=1732029304381 2024-11-19T15:15:05,907 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#B#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:05,908 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/4cf25ca6d66942ddb5fa0764f4cec024 is 50, key is test_row_0/B:col10/1732029304399/Put/seqid=0 2024-11-19T15:15:05,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742159_1335 (size=13357) 2024-11-19T15:15:05,915 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#A#compaction#291 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:05,916 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/a79214ef29f0480b8fa92d4f9932731d is 50, key is test_row_0/A:col10/1732029304399/Put/seqid=0 2024-11-19T15:15:05,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742160_1336 (size=13357) 2024-11-19T15:15:05,921 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/4cf25ca6d66942ddb5fa0764f4cec024 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4cf25ca6d66942ddb5fa0764f4cec024 2024-11-19T15:15:05,926 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/a79214ef29f0480b8fa92d4f9932731d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/a79214ef29f0480b8fa92d4f9932731d 2024-11-19T15:15:05,926 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/B of bd5712f88c144f357fc59afd6d3ab798 into 4cf25ca6d66942ddb5fa0764f4cec024(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:05,926 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:05,926 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/B, priority=13, startTime=1732029305898; duration=0sec 2024-11-19T15:15:05,926 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:05,926 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:B 2024-11-19T15:15:05,927 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:15:05,928 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47646 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:15:05,928 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): bd5712f88c144f357fc59afd6d3ab798/C is initiating minor compaction (all files) 2024-11-19T15:15:05,928 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bd5712f88c144f357fc59afd6d3ab798/C in TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:05,928 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/e99ca1d7bb704068b197e25a750f8f23, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/035a29970f4f48ffab2f4c8d340f1221, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/ddc6a67768ab4a8caac5eb9e5593719f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/671719e0baa34898b65a814227a66466] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp, totalSize=46.5 K 2024-11-19T15:15:05,929 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting e99ca1d7bb704068b197e25a750f8f23, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1732029301971 2024-11-19T15:15:05,929 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 035a29970f4f48ffab2f4c8d340f1221, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=620, earliestPutTs=1732029303134 2024-11-19T15:15:05,929 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting ddc6a67768ab4a8caac5eb9e5593719f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1732029303211 2024-11-19T15:15:05,930 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 671719e0baa34898b65a814227a66466, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=673, earliestPutTs=1732029304381 2024-11-19T15:15:05,931 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/A of bd5712f88c144f357fc59afd6d3ab798 into a79214ef29f0480b8fa92d4f9932731d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:05,931 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:05,931 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/A, priority=13, startTime=1732029305898; duration=0sec 2024-11-19T15:15:05,931 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:05,931 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:A 2024-11-19T15:15:05,938 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bd5712f88c144f357fc59afd6d3ab798#C#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:05,939 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/b90912da8e0b4c59a1cac2314b53c365 is 50, key is test_row_0/C:col10/1732029304399/Put/seqid=0 2024-11-19T15:15:05,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742161_1337 (size=13323) 2024-11-19T15:15:06,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T15:15:06,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:06,023 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T15:15:06,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:15:06,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:06,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:15:06,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:06,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:15:06,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:06,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/6289f79a65614947bbee62fdc44ab872 is 50, key is test_row_0/A:col10/1732029305064/Put/seqid=0 2024-11-19T15:15:06,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742162_1338 (size=12301) 2024-11-19T15:15:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:15:06,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. as already flushing 2024-11-19T15:15:06,226 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-19T15:15:06,226 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:06,226 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:06,226 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:06,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:06,235 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-11-19T15:15:06,235 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:06,235 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:06,236 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:06,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:06,347 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/b90912da8e0b4c59a1cac2314b53c365 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/b90912da8e0b4c59a1cac2314b53c365 2024-11-19T15:15:06,352 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bd5712f88c144f357fc59afd6d3ab798/C of bd5712f88c144f357fc59afd6d3ab798 into b90912da8e0b4c59a1cac2314b53c365(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:06,352 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:06,352 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., storeName=bd5712f88c144f357fc59afd6d3ab798/C, priority=12, startTime=1732029305898; duration=0sec 2024-11-19T15:15:06,352 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:06,352 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bd5712f88c144f357fc59afd6d3ab798:C 2024-11-19T15:15:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T15:15:06,431 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/6289f79a65614947bbee62fdc44ab872 2024-11-19T15:15:06,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/899e64df763d40fda03541c524bb9777 is 50, key is test_row_0/B:col10/1732029305064/Put/seqid=0 2024-11-19T15:15:06,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742163_1339 (size=12301) 2024-11-19T15:15:06,843 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/899e64df763d40fda03541c524bb9777 2024-11-19T15:15:06,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/a4bdf3eacd1140ecac7a7133933fea03 is 50, key is test_row_0/C:col10/1732029305064/Put/seqid=0 2024-11-19T15:15:06,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742164_1340 (size=12301) 2024-11-19T15:15:06,979 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T15:15:07,254 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/a4bdf3eacd1140ecac7a7133933fea03 2024-11-19T15:15:07,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/6289f79a65614947bbee62fdc44ab872 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/6289f79a65614947bbee62fdc44ab872 2024-11-19T15:15:07,261 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/6289f79a65614947bbee62fdc44ab872, entries=150, sequenceid=695, filesize=12.0 K 2024-11-19T15:15:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/899e64df763d40fda03541c524bb9777 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/899e64df763d40fda03541c524bb9777 2024-11-19T15:15:07,265 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/899e64df763d40fda03541c524bb9777, entries=150, sequenceid=695, filesize=12.0 K 2024-11-19T15:15:07,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/a4bdf3eacd1140ecac7a7133933fea03 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a4bdf3eacd1140ecac7a7133933fea03 2024-11-19T15:15:07,269 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a4bdf3eacd1140ecac7a7133933fea03, entries=150, sequenceid=695, filesize=12.0 K 2024-11-19T15:15:07,270 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for bd5712f88c144f357fc59afd6d3ab798 in 1247ms, sequenceid=695, compaction requested=false 2024-11-19T15:15:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for bd5712f88c144f357fc59afd6d3ab798: 2024-11-19T15:15:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-19T15:15:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-19T15:15:07,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-19T15:15:07,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0140 sec 2024-11-19T15:15:07,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 2.0190 sec 2024-11-19T15:15:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T15:15:07,404 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:14,675 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-11-19T15:15:14,675 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:14,675 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:14,675 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:14,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:14,685 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-11-19T15:15:14,685 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:14,685 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:14,685 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:14,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:14,695 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-11-19T15:15:14,695 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:14,695 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:14,695 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:14,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:14,695 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T15:15:14,695 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 294 2024-11-19T15:15:14,695 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 255 2024-11-19T15:15:14,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-19T15:15:14,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-19T15:15:14,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-19T15:15:14,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T15:15:14,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3202 2024-11-19T15:15:14,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3288 2024-11-19T15:15:14,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3114 2024-11-19T15:15:14,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3206 2024-11-19T15:15:14,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3297 2024-11-19T15:15:14,696 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T15:15:14,696 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T15:15:14,696 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T15:15:14,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:14,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:14,696 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:14,697 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T15:15:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T15:15:14,699 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029314699"}]},"ts":"1732029314699"} 2024-11-19T15:15:14,700 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T15:15:14,700 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T15:15:14,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T15:15:14,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd5712f88c144f357fc59afd6d3ab798, UNASSIGN}] 2024-11-19T15:15:14,703 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd5712f88c144f357fc59afd6d3ab798, UNASSIGN 2024-11-19T15:15:14,703 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=bd5712f88c144f357fc59afd6d3ab798, regionState=CLOSING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:14,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd5712f88c144f357fc59afd6d3ab798, UNASSIGN because future has completed 2024-11-19T15:15:14,705 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T15:15:14,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:15:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T15:15:14,857 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:15:14,857 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T15:15:14,857 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing bd5712f88c144f357fc59afd6d3ab798, disabling compactions & flushes 2024-11-19T15:15:14,857 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:14,857 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:14,857 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. after waiting 0 ms 2024-11-19T15:15:14,857 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:14,857 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing bd5712f88c144f357fc59afd6d3ab798 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T15:15:14,857 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=A 2024-11-19T15:15:14,857 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:14,857 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=B 2024-11-19T15:15:14,857 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:14,857 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bd5712f88c144f357fc59afd6d3ab798, store=C 2024-11-19T15:15:14,857 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:14,861 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/73932c0606dc4a728e4be9121daabbd3 is 50, key is test_row_0/A:col10/1732029314684/Put/seqid=0 2024-11-19T15:15:14,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742165_1341 (size=12301) 2024-11-19T15:15:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T15:15:15,265 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/73932c0606dc4a728e4be9121daabbd3 2024-11-19T15:15:15,271 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/ed79a75a7f3f4685bcdf9505bb5846f9 is 50, key is test_row_0/B:col10/1732029314684/Put/seqid=0 2024-11-19T15:15:15,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742166_1342 (size=12301) 2024-11-19T15:15:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T15:15:15,675 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/ed79a75a7f3f4685bcdf9505bb5846f9 2024-11-19T15:15:15,681 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/a5cecc00dabf4e368b5c0a564faa5d75 is 50, key is test_row_0/C:col10/1732029314684/Put/seqid=0 2024-11-19T15:15:15,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742167_1343 (size=12301) 2024-11-19T15:15:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T15:15:16,084 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/a5cecc00dabf4e368b5c0a564faa5d75 2024-11-19T15:15:16,088 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/A/73932c0606dc4a728e4be9121daabbd3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/73932c0606dc4a728e4be9121daabbd3 2024-11-19T15:15:16,091 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/73932c0606dc4a728e4be9121daabbd3, entries=150, sequenceid=704, filesize=12.0 K 2024-11-19T15:15:16,091 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/B/ed79a75a7f3f4685bcdf9505bb5846f9 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/ed79a75a7f3f4685bcdf9505bb5846f9 2024-11-19T15:15:16,094 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/ed79a75a7f3f4685bcdf9505bb5846f9, entries=150, sequenceid=704, filesize=12.0 K 2024-11-19T15:15:16,095 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/.tmp/C/a5cecc00dabf4e368b5c0a564faa5d75 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a5cecc00dabf4e368b5c0a564faa5d75 2024-11-19T15:15:16,097 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a5cecc00dabf4e368b5c0a564faa5d75, entries=150, sequenceid=704, filesize=12.0 K 2024-11-19T15:15:16,098 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for bd5712f88c144f357fc59afd6d3ab798 in 1241ms, sequenceid=704, compaction requested=true 2024-11-19T15:15:16,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b31654aef239421daae93e3a475b336b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e00bfbbc8488461a9b4666ebcd0b6616, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/cc9badbe797f44d2a414802b84fed854, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/beb4271011f542e7a6436a7218b8a68e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/3091c3eaaa234740af2e4f37bc5d2ac6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/c358b38942bf411c9ef62a088ccb95f0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/72f272c1bef743afa51d9f11d3b29357, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/0f1b46dd8a4042caa2b7b4d6b346df54, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/9c525f98cddb4e3eac1fd412f7f2b0c7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/2c47e34c28aa4fe785c69ce6fdd6a5b5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/de1726cea1ce4b56b5113014ea2d96a1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e0d9cfa17c5c4f549003bf18244b847f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/373a470682df4992bb2d301a120572ea, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/c2e5c583d85148908ac94f62bc25e391, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b8909d92f1fe4b159735d3c3deeb1a3c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/5a7c512ca1334c53944e48c3413ff08c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/4be5e23dc1b740b0913299ebaef8455b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/bb237def9a314cf9aef4506c8ef5585f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b104ae8698584f359da9a33a1f8fb75d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/bd2d0ea4f62c4961a2d402191deb6eb1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/092393d21ae04a1da71203041e606032, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/34cd3b0bb7254064b786b8aa97783063, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e21b25cb837f41d08c7e453b0851a705, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/fcc5c5137a3548ae9b61bc5a06ddcc4c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e36b6f1eb6dd4791bea42e0f4b0ab333, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e1f87d5f530a40c8ad91c4b30e1d1aa8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/37cdd778ddeb4abea833cbf25d4966c4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/142057b5c2bb4672aeec53f0295a89eb, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/59ab0d4b063e40d4b5244f65996c31af, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/67ae24b4747943e1a3fc8fc70416e947, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/0f416444b99f489f89564be7868437a4] to archive 2024-11-19T15:15:16,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:15:16,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b31654aef239421daae93e3a475b336b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b31654aef239421daae93e3a475b336b 2024-11-19T15:15:16,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e00bfbbc8488461a9b4666ebcd0b6616 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e00bfbbc8488461a9b4666ebcd0b6616 2024-11-19T15:15:16,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/cc9badbe797f44d2a414802b84fed854 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/cc9badbe797f44d2a414802b84fed854 2024-11-19T15:15:16,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/beb4271011f542e7a6436a7218b8a68e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/beb4271011f542e7a6436a7218b8a68e 2024-11-19T15:15:16,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/3091c3eaaa234740af2e4f37bc5d2ac6 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/3091c3eaaa234740af2e4f37bc5d2ac6 2024-11-19T15:15:16,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/c358b38942bf411c9ef62a088ccb95f0 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/c358b38942bf411c9ef62a088ccb95f0 2024-11-19T15:15:16,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/72f272c1bef743afa51d9f11d3b29357 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/72f272c1bef743afa51d9f11d3b29357 2024-11-19T15:15:16,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/0f1b46dd8a4042caa2b7b4d6b346df54 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/0f1b46dd8a4042caa2b7b4d6b346df54 2024-11-19T15:15:16,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/9c525f98cddb4e3eac1fd412f7f2b0c7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/9c525f98cddb4e3eac1fd412f7f2b0c7 2024-11-19T15:15:16,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/2c47e34c28aa4fe785c69ce6fdd6a5b5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/2c47e34c28aa4fe785c69ce6fdd6a5b5 2024-11-19T15:15:16,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/de1726cea1ce4b56b5113014ea2d96a1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/de1726cea1ce4b56b5113014ea2d96a1 2024-11-19T15:15:16,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e0d9cfa17c5c4f549003bf18244b847f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e0d9cfa17c5c4f549003bf18244b847f 2024-11-19T15:15:16,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/373a470682df4992bb2d301a120572ea to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/373a470682df4992bb2d301a120572ea 2024-11-19T15:15:16,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/c2e5c583d85148908ac94f62bc25e391 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/c2e5c583d85148908ac94f62bc25e391 2024-11-19T15:15:16,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b8909d92f1fe4b159735d3c3deeb1a3c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b8909d92f1fe4b159735d3c3deeb1a3c 2024-11-19T15:15:16,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/5a7c512ca1334c53944e48c3413ff08c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/5a7c512ca1334c53944e48c3413ff08c 2024-11-19T15:15:16,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/4be5e23dc1b740b0913299ebaef8455b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/4be5e23dc1b740b0913299ebaef8455b 2024-11-19T15:15:16,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/bb237def9a314cf9aef4506c8ef5585f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/bb237def9a314cf9aef4506c8ef5585f 2024-11-19T15:15:16,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b104ae8698584f359da9a33a1f8fb75d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/b104ae8698584f359da9a33a1f8fb75d 2024-11-19T15:15:16,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/bd2d0ea4f62c4961a2d402191deb6eb1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/bd2d0ea4f62c4961a2d402191deb6eb1 2024-11-19T15:15:16,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/092393d21ae04a1da71203041e606032 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/092393d21ae04a1da71203041e606032 2024-11-19T15:15:16,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/34cd3b0bb7254064b786b8aa97783063 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/34cd3b0bb7254064b786b8aa97783063 2024-11-19T15:15:16,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e21b25cb837f41d08c7e453b0851a705 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e21b25cb837f41d08c7e453b0851a705 2024-11-19T15:15:16,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/fcc5c5137a3548ae9b61bc5a06ddcc4c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/fcc5c5137a3548ae9b61bc5a06ddcc4c 2024-11-19T15:15:16,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e36b6f1eb6dd4791bea42e0f4b0ab333 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e36b6f1eb6dd4791bea42e0f4b0ab333 2024-11-19T15:15:16,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e1f87d5f530a40c8ad91c4b30e1d1aa8 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/e1f87d5f530a40c8ad91c4b30e1d1aa8 2024-11-19T15:15:16,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/37cdd778ddeb4abea833cbf25d4966c4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/37cdd778ddeb4abea833cbf25d4966c4 2024-11-19T15:15:16,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/142057b5c2bb4672aeec53f0295a89eb to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/142057b5c2bb4672aeec53f0295a89eb 2024-11-19T15:15:16,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/59ab0d4b063e40d4b5244f65996c31af to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/59ab0d4b063e40d4b5244f65996c31af 2024-11-19T15:15:16,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/67ae24b4747943e1a3fc8fc70416e947 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/67ae24b4747943e1a3fc8fc70416e947 2024-11-19T15:15:16,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/0f416444b99f489f89564be7868437a4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/0f416444b99f489f89564be7868437a4 2024-11-19T15:15:16,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/596dbd0d1d8a48ddbedf1f35b8363ad2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/63fb6d90f3214e0f9cfdb309f56694d5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/e5b6337c5cb144febb5c86cf7b033509, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4fe115ca0b4848c48a1a90010a807964, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/8a044c518bf44c9f96407e834722b070, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/57ff9c24db9247a6a5badb3451569775, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/99c1f3d2478041d2aa6e3a8c27ab429f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/7802b3e03b8b45bd972efa9f6b34ac01, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/674400ff4d5b4c4f82be917ea92cb768, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/9098e7d94b0b47a3891a5732d38e2c43, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0d0c7246cc9d4f6a85f7d746e8d3c32b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/ade8df61c11d49df8bd909ddbbb2e8c2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0cda22b4dc904ab19e3fb86059b96c13, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/918841a0bf4e46808d62fe22ffea1dea, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/b6b71dc088ab43bfb6033b557ef399ca, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/afa4228ea45b462e843216f89a587d39, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/1b432df1bf1143a2912701c375f4a4b5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/c3039f3c66314bb097e5f28405b9d69a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/301819fd52c54adcbfb124df45477fc3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/3beace3e657442809c34bcea95bdd1a4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0b31d104646f403fa4554f2472e4f2d4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/6ab2db1a46d14da793f1d3acc4a3afc2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4be4cb0ed4934ea0a168a04a2050a9ad, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/d5988d4dd43045749510a7900c663941, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/75dd0976a3554a57bb8ecfb0753a3c5f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/fae504a1a6054195a7556b759b219b70, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/1d2376ffaa784ddda054fb2ea41cb001, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/9da3ed874e5e440f8eebac8d73865ae8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/7a575dd8dc9d4c67ab05b0997037624f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/f00f45481d594e2cb95251e8ba6c3af3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/98e8cd11e2354050acfa9f4ca3652376] to archive 2024-11-19T15:15:16,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:15:16,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/596dbd0d1d8a48ddbedf1f35b8363ad2 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/596dbd0d1d8a48ddbedf1f35b8363ad2 2024-11-19T15:15:16,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/63fb6d90f3214e0f9cfdb309f56694d5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/63fb6d90f3214e0f9cfdb309f56694d5 2024-11-19T15:15:16,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/e5b6337c5cb144febb5c86cf7b033509 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/e5b6337c5cb144febb5c86cf7b033509 2024-11-19T15:15:16,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4fe115ca0b4848c48a1a90010a807964 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4fe115ca0b4848c48a1a90010a807964 2024-11-19T15:15:16,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/8a044c518bf44c9f96407e834722b070 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/8a044c518bf44c9f96407e834722b070 2024-11-19T15:15:16,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/57ff9c24db9247a6a5badb3451569775 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/57ff9c24db9247a6a5badb3451569775 2024-11-19T15:15:16,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/99c1f3d2478041d2aa6e3a8c27ab429f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/99c1f3d2478041d2aa6e3a8c27ab429f 2024-11-19T15:15:16,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/7802b3e03b8b45bd972efa9f6b34ac01 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/7802b3e03b8b45bd972efa9f6b34ac01 2024-11-19T15:15:16,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/674400ff4d5b4c4f82be917ea92cb768 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/674400ff4d5b4c4f82be917ea92cb768 2024-11-19T15:15:16,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/9098e7d94b0b47a3891a5732d38e2c43 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/9098e7d94b0b47a3891a5732d38e2c43 2024-11-19T15:15:16,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0d0c7246cc9d4f6a85f7d746e8d3c32b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0d0c7246cc9d4f6a85f7d746e8d3c32b 2024-11-19T15:15:16,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/ade8df61c11d49df8bd909ddbbb2e8c2 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/ade8df61c11d49df8bd909ddbbb2e8c2 2024-11-19T15:15:16,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0cda22b4dc904ab19e3fb86059b96c13 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0cda22b4dc904ab19e3fb86059b96c13 2024-11-19T15:15:16,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/918841a0bf4e46808d62fe22ffea1dea to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/918841a0bf4e46808d62fe22ffea1dea 2024-11-19T15:15:16,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/b6b71dc088ab43bfb6033b557ef399ca to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/b6b71dc088ab43bfb6033b557ef399ca 2024-11-19T15:15:16,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/afa4228ea45b462e843216f89a587d39 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/afa4228ea45b462e843216f89a587d39 2024-11-19T15:15:16,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/1b432df1bf1143a2912701c375f4a4b5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/1b432df1bf1143a2912701c375f4a4b5 2024-11-19T15:15:16,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/c3039f3c66314bb097e5f28405b9d69a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/c3039f3c66314bb097e5f28405b9d69a 2024-11-19T15:15:16,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/301819fd52c54adcbfb124df45477fc3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/301819fd52c54adcbfb124df45477fc3 2024-11-19T15:15:16,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/3beace3e657442809c34bcea95bdd1a4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/3beace3e657442809c34bcea95bdd1a4 2024-11-19T15:15:16,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0b31d104646f403fa4554f2472e4f2d4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/0b31d104646f403fa4554f2472e4f2d4 2024-11-19T15:15:16,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/6ab2db1a46d14da793f1d3acc4a3afc2 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/6ab2db1a46d14da793f1d3acc4a3afc2 2024-11-19T15:15:16,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4be4cb0ed4934ea0a168a04a2050a9ad to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4be4cb0ed4934ea0a168a04a2050a9ad 2024-11-19T15:15:16,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/d5988d4dd43045749510a7900c663941 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/d5988d4dd43045749510a7900c663941 2024-11-19T15:15:16,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/75dd0976a3554a57bb8ecfb0753a3c5f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/75dd0976a3554a57bb8ecfb0753a3c5f 2024-11-19T15:15:16,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/fae504a1a6054195a7556b759b219b70 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/fae504a1a6054195a7556b759b219b70 2024-11-19T15:15:16,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/1d2376ffaa784ddda054fb2ea41cb001 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/1d2376ffaa784ddda054fb2ea41cb001 2024-11-19T15:15:16,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/9da3ed874e5e440f8eebac8d73865ae8 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/9da3ed874e5e440f8eebac8d73865ae8 2024-11-19T15:15:16,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/7a575dd8dc9d4c67ab05b0997037624f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/7a575dd8dc9d4c67ab05b0997037624f 2024-11-19T15:15:16,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/f00f45481d594e2cb95251e8ba6c3af3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/f00f45481d594e2cb95251e8ba6c3af3 2024-11-19T15:15:16,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/98e8cd11e2354050acfa9f4ca3652376 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/98e8cd11e2354050acfa9f4ca3652376 2024-11-19T15:15:16,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/8849575653a54682836ca065b4f631b4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d581c816a4f847c5bb74cbc5fcf11743, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/e82fa89591e541b49c722f44570a761e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/1403f48dbad54d8aad16fb1dd2ac7937, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d6377693396e48d6a6833d54494d9ac1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/7a109ee8a70942a0a99fc2cb2acf7e8c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a1939588f351404f8e046a9c92ae4bfe, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/019bae089d8243a088609d6dcf03349d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/556b6ae71a3e47a19e9e3179a253f1f7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/59560d9455af49848b98e45cb0ec66af, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/9ad53a102aec4522ad1a2616a3a083a6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/893c16256cff4339a63343b7ed6580ab, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/44f099879c174d2a8c9d95baa8a58604, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/18e903aacbdb436199ab597e3f765b8f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/4e8b6aa3eacb4d91bcc17b50d42e0457, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/dad34d0237564eb8b06271525caee1e7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/3786ed2918634132adcc8645cdc29eca, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/3846432e85744767933dfce73784d1f7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/1480beacae8b49b281d6fd741884b746, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/55336742c0c74eadb29064805c766fb6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d82c2f1e37a048679f6d31cbd96219f9, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/14a4c2bbecec43dfbebbecdbc23a1936, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/28baff02eeac4cbbb94704f10d74c1e3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d47996751412460db8ac05e45af23be1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d8dff4fd0e984345bdefbd251a35903a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/e99ca1d7bb704068b197e25a750f8f23, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/25fd548d3638463b9f22ebd61bbe8db5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/035a29970f4f48ffab2f4c8d340f1221, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/ddc6a67768ab4a8caac5eb9e5593719f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/671719e0baa34898b65a814227a66466] to archive 2024-11-19T15:15:16,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:15:16,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/8849575653a54682836ca065b4f631b4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/8849575653a54682836ca065b4f631b4 2024-11-19T15:15:16,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d581c816a4f847c5bb74cbc5fcf11743 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d581c816a4f847c5bb74cbc5fcf11743 2024-11-19T15:15:16,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/e82fa89591e541b49c722f44570a761e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/e82fa89591e541b49c722f44570a761e 2024-11-19T15:15:16,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/1403f48dbad54d8aad16fb1dd2ac7937 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/1403f48dbad54d8aad16fb1dd2ac7937 2024-11-19T15:15:16,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d6377693396e48d6a6833d54494d9ac1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d6377693396e48d6a6833d54494d9ac1 2024-11-19T15:15:16,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/7a109ee8a70942a0a99fc2cb2acf7e8c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/7a109ee8a70942a0a99fc2cb2acf7e8c 2024-11-19T15:15:16,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a1939588f351404f8e046a9c92ae4bfe to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a1939588f351404f8e046a9c92ae4bfe 2024-11-19T15:15:16,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/019bae089d8243a088609d6dcf03349d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/019bae089d8243a088609d6dcf03349d 2024-11-19T15:15:16,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/556b6ae71a3e47a19e9e3179a253f1f7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/556b6ae71a3e47a19e9e3179a253f1f7 2024-11-19T15:15:16,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/59560d9455af49848b98e45cb0ec66af to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/59560d9455af49848b98e45cb0ec66af 2024-11-19T15:15:16,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/9ad53a102aec4522ad1a2616a3a083a6 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/9ad53a102aec4522ad1a2616a3a083a6 2024-11-19T15:15:16,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/893c16256cff4339a63343b7ed6580ab to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/893c16256cff4339a63343b7ed6580ab 2024-11-19T15:15:16,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/44f099879c174d2a8c9d95baa8a58604 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/44f099879c174d2a8c9d95baa8a58604 2024-11-19T15:15:16,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/18e903aacbdb436199ab597e3f765b8f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/18e903aacbdb436199ab597e3f765b8f 2024-11-19T15:15:16,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/4e8b6aa3eacb4d91bcc17b50d42e0457 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/4e8b6aa3eacb4d91bcc17b50d42e0457 2024-11-19T15:15:16,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/dad34d0237564eb8b06271525caee1e7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/dad34d0237564eb8b06271525caee1e7 2024-11-19T15:15:16,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/3786ed2918634132adcc8645cdc29eca to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/3786ed2918634132adcc8645cdc29eca 2024-11-19T15:15:16,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/3846432e85744767933dfce73784d1f7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/3846432e85744767933dfce73784d1f7 2024-11-19T15:15:16,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/1480beacae8b49b281d6fd741884b746 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/1480beacae8b49b281d6fd741884b746 2024-11-19T15:15:16,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/55336742c0c74eadb29064805c766fb6 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/55336742c0c74eadb29064805c766fb6 2024-11-19T15:15:16,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d82c2f1e37a048679f6d31cbd96219f9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d82c2f1e37a048679f6d31cbd96219f9 2024-11-19T15:15:16,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/14a4c2bbecec43dfbebbecdbc23a1936 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/14a4c2bbecec43dfbebbecdbc23a1936 2024-11-19T15:15:16,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/28baff02eeac4cbbb94704f10d74c1e3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/28baff02eeac4cbbb94704f10d74c1e3 2024-11-19T15:15:16,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d47996751412460db8ac05e45af23be1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d47996751412460db8ac05e45af23be1 2024-11-19T15:15:16,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d8dff4fd0e984345bdefbd251a35903a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/d8dff4fd0e984345bdefbd251a35903a 2024-11-19T15:15:16,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/e99ca1d7bb704068b197e25a750f8f23 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/e99ca1d7bb704068b197e25a750f8f23 2024-11-19T15:15:16,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/25fd548d3638463b9f22ebd61bbe8db5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/25fd548d3638463b9f22ebd61bbe8db5 2024-11-19T15:15:16,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/035a29970f4f48ffab2f4c8d340f1221 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/035a29970f4f48ffab2f4c8d340f1221 2024-11-19T15:15:16,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/ddc6a67768ab4a8caac5eb9e5593719f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/ddc6a67768ab4a8caac5eb9e5593719f 2024-11-19T15:15:16,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/671719e0baa34898b65a814227a66466 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/671719e0baa34898b65a814227a66466 2024-11-19T15:15:16,189 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/recovered.edits/707.seqid, newMaxSeqId=707, maxSeqId=1 2024-11-19T15:15:16,189 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. 2024-11-19T15:15:16,189 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for bd5712f88c144f357fc59afd6d3ab798: Waiting for close lock at 1732029314857Running coprocessor pre-close hooks at 1732029314857Disabling compacts and flushes for region at 1732029314857Disabling writes for close at 1732029314857Obtaining lock to block concurrent updates at 1732029314857Preparing flush snapshotting stores in bd5712f88c144f357fc59afd6d3ab798 at 1732029314857Finished memstore snapshotting TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732029314857Flushing stores of TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798. at 1732029314858 (+1 ms)Flushing bd5712f88c144f357fc59afd6d3ab798/A: creating writer at 1732029314858Flushing bd5712f88c144f357fc59afd6d3ab798/A: appending metadata at 1732029314860 (+2 ms)Flushing bd5712f88c144f357fc59afd6d3ab798/A: closing flushed file at 1732029314860Flushing bd5712f88c144f357fc59afd6d3ab798/B: creating writer at 1732029315269 (+409 ms)Flushing bd5712f88c144f357fc59afd6d3ab798/B: appending metadata at 1732029315271 (+2 ms)Flushing bd5712f88c144f357fc59afd6d3ab798/B: closing flushed file at 1732029315271Flushing bd5712f88c144f357fc59afd6d3ab798/C: creating writer at 1732029315678 (+407 ms)Flushing bd5712f88c144f357fc59afd6d3ab798/C: appending metadata at 1732029315680 (+2 ms)Flushing bd5712f88c144f357fc59afd6d3ab798/C: closing flushed file at 1732029315680Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@324fdeaa: reopening flushed file at 1732029316087 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26622b9a: reopening flushed file at 1732029316091 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1da8a8fc: reopening flushed file at 1732029316094 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for bd5712f88c144f357fc59afd6d3ab798 in 1241ms, sequenceid=704, compaction requested=true at 1732029316098 (+4 ms)Writing region close event to WAL at 1732029316186 (+88 ms)Running coprocessor post-close hooks at 1732029316189 (+3 ms)Closed at 1732029316189 2024-11-19T15:15:16,191 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:15:16,192 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=bd5712f88c144f357fc59afd6d3ab798, regionState=CLOSED 2024-11-19T15:15:16,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:15:16,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-19T15:15:16,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure bd5712f88c144f357fc59afd6d3ab798, server=50570b5b74e7,46557,1732029218639 in 1.4890 sec 2024-11-19T15:15:16,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-19T15:15:16,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bd5712f88c144f357fc59afd6d3ab798, UNASSIGN in 1.4930 sec 2024-11-19T15:15:16,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-19T15:15:16,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4960 sec 2024-11-19T15:15:16,199 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029316199"}]},"ts":"1732029316199"} 2024-11-19T15:15:16,200 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T15:15:16,200 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T15:15:16,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5040 sec 2024-11-19T15:15:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T15:15:16,844 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:16,845 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T15:15:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:16,846 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T15:15:16,847 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:16,849 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:15:16,851 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/recovered.edits] 2024-11-19T15:15:16,854 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/6289f79a65614947bbee62fdc44ab872 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/6289f79a65614947bbee62fdc44ab872 2024-11-19T15:15:16,855 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/73932c0606dc4a728e4be9121daabbd3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/73932c0606dc4a728e4be9121daabbd3 2024-11-19T15:15:16,856 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/a79214ef29f0480b8fa92d4f9932731d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/A/a79214ef29f0480b8fa92d4f9932731d 2024-11-19T15:15:16,858 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4cf25ca6d66942ddb5fa0764f4cec024 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/4cf25ca6d66942ddb5fa0764f4cec024 2024-11-19T15:15:16,859 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/899e64df763d40fda03541c524bb9777 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/899e64df763d40fda03541c524bb9777 2024-11-19T15:15:16,859 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/ed79a75a7f3f4685bcdf9505bb5846f9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/B/ed79a75a7f3f4685bcdf9505bb5846f9 2024-11-19T15:15:16,861 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a4bdf3eacd1140ecac7a7133933fea03 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a4bdf3eacd1140ecac7a7133933fea03 2024-11-19T15:15:16,862 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a5cecc00dabf4e368b5c0a564faa5d75 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/a5cecc00dabf4e368b5c0a564faa5d75 2024-11-19T15:15:16,863 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/b90912da8e0b4c59a1cac2314b53c365 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/C/b90912da8e0b4c59a1cac2314b53c365 2024-11-19T15:15:16,866 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/recovered.edits/707.seqid to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798/recovered.edits/707.seqid 2024-11-19T15:15:16,866 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/bd5712f88c144f357fc59afd6d3ab798 2024-11-19T15:15:16,866 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T15:15:16,868 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:16,871 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T15:15:16,872 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T15:15:16,873 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:16,873 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T15:15:16,874 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732029316873"}]},"ts":"9223372036854775807"} 2024-11-19T15:15:16,875 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T15:15:16,875 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bd5712f88c144f357fc59afd6d3ab798, NAME => 'TestAcidGuarantees,,1732029284570.bd5712f88c144f357fc59afd6d3ab798.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T15:15:16,875 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T15:15:16,875 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732029316875"}]},"ts":"9223372036854775807"} 2024-11-19T15:15:16,877 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T15:15:16,877 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:16,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-19T15:15:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T15:15:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T15:15:16,955 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:16,964 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=245 (was 247), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=414 (was 482), ProcessCount=11 (was 11), AvailableMemoryMB=7873 (was 8236) 2024-11-19T15:15:16,972 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=245, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=414, ProcessCount=11, AvailableMemoryMB=7873 2024-11-19T15:15:16,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T15:15:16,974 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T15:15:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:16,976 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T15:15:16,976 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:16,976 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-19T15:15:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T15:15:16,977 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T15:15:16,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742168_1344 (size=1061) 2024-11-19T15:15:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T15:15:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T15:15:17,384 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8 2024-11-19T15:15:17,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742169_1345 (size=53) 2024-11-19T15:15:17,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:15:17,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 09d12e268a82f5728962d61ed7bcb9c9, disabling compactions & flushes 2024-11-19T15:15:17,390 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:17,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:17,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. after waiting 0 ms 2024-11-19T15:15:17,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:17,390 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:17,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 09d12e268a82f5728962d61ed7bcb9c9: Waiting for close lock at 1732029317390Disabling compacts and flushes for region at 1732029317390Disabling writes for close at 1732029317390Writing region close event to WAL at 1732029317390Closed at 1732029317390 2024-11-19T15:15:17,391 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T15:15:17,391 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732029317391"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732029317391"}]},"ts":"1732029317391"} 2024-11-19T15:15:17,393 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T15:15:17,394 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T15:15:17,394 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029317394"}]},"ts":"1732029317394"} 2024-11-19T15:15:17,395 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T15:15:17,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, ASSIGN}] 2024-11-19T15:15:17,397 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, ASSIGN 2024-11-19T15:15:17,398 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, ASSIGN; state=OFFLINE, location=50570b5b74e7,46557,1732029218639; forceNewPlan=false, retain=false 2024-11-19T15:15:17,549 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=09d12e268a82f5728962d61ed7bcb9c9, regionState=OPENING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:17,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, ASSIGN because future has completed 2024-11-19T15:15:17,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:15:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T15:15:17,707 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:17,707 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} 2024-11-19T15:15:17,707 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,707 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:15:17,708 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,708 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,709 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,710 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:15:17,710 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09d12e268a82f5728962d61ed7bcb9c9 columnFamilyName A 2024-11-19T15:15:17,710 DEBUG [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:17,710 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.HStore(327): Store=09d12e268a82f5728962d61ed7bcb9c9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:15:17,711 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,711 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:15:17,711 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09d12e268a82f5728962d61ed7bcb9c9 columnFamilyName B 2024-11-19T15:15:17,711 DEBUG [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:17,712 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.HStore(327): Store=09d12e268a82f5728962d61ed7bcb9c9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:15:17,712 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,713 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:15:17,713 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09d12e268a82f5728962d61ed7bcb9c9 columnFamilyName C 2024-11-19T15:15:17,713 DEBUG [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:17,713 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.HStore(327): Store=09d12e268a82f5728962d61ed7bcb9c9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:15:17,713 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:17,713 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,714 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,714 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,715 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,715 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,715 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T15:15:17,716 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,718 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T15:15:17,718 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 09d12e268a82f5728962d61ed7bcb9c9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63109416, jitterRate=-0.05959641933441162}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T15:15:17,718 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:17,718 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 09d12e268a82f5728962d61ed7bcb9c9: Running coprocessor pre-open hook at 1732029317708Writing region info on filesystem at 1732029317708Initializing all the Stores at 1732029317708Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029317708Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029317709 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029317709Cleaning up temporary data from old regions at 1732029317715 (+6 ms)Running coprocessor post-open hooks at 1732029317718 (+3 ms)Region opened successfully at 1732029317718 2024-11-19T15:15:17,719 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., pid=95, masterSystemTime=1732029317704 2024-11-19T15:15:17,721 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:17,721 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:17,721 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=09d12e268a82f5728962d61ed7bcb9c9, regionState=OPEN, openSeqNum=2, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:17,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:15:17,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-19T15:15:17,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 in 171 msec 2024-11-19T15:15:17,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-19T15:15:17,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, ASSIGN in 330 msec 2024-11-19T15:15:17,727 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T15:15:17,727 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029317727"}]},"ts":"1732029317727"} 2024-11-19T15:15:17,728 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T15:15:17,729 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T15:15:17,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 755 msec 2024-11-19T15:15:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T15:15:18,121 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:18,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3de22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:18,127 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:18,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:18,130 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:18,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:18,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733569c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:18,134 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:18,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a73ba3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:18,135 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:18,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:18,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:18,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:18,141 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T15:15:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 50570b5b74e7,39501,1732029217890 2024-11-19T15:15:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@401291c9 2024-11-19T15:15:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T15:15:18,143 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T15:15:18,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T15:15:18,146 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T15:15:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:18,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742170_1346 (size=1097) 2024-11-19T15:15:18,568 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T15:15:18,568 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T15:15:18,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T15:15:18,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, REOPEN/MOVE}] 2024-11-19T15:15:18,575 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, REOPEN/MOVE 2024-11-19T15:15:18,576 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=09d12e268a82f5728962d61ed7bcb9c9, regionState=CLOSING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:18,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, REOPEN/MOVE because future has completed 2024-11-19T15:15:18,578 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T15:15:18,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:15:18,731 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:18,731 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T15:15:18,731 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 09d12e268a82f5728962d61ed7bcb9c9, disabling compactions & flushes 2024-11-19T15:15:18,731 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:18,731 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:18,731 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. after waiting 0 ms 2024-11-19T15:15:18,731 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:18,735 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T15:15:18,736 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:18,736 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 09d12e268a82f5728962d61ed7bcb9c9: Waiting for close lock at 1732029318731Running coprocessor pre-close hooks at 1732029318731Disabling compacts and flushes for region at 1732029318731Disabling writes for close at 1732029318731Writing region close event to WAL at 1732029318732 (+1 ms)Running coprocessor post-close hooks at 1732029318736 (+4 ms)Closed at 1732029318736 2024-11-19T15:15:18,736 WARN [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 09d12e268a82f5728962d61ed7bcb9c9 to self. 2024-11-19T15:15:18,738 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:18,739 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=09d12e268a82f5728962d61ed7bcb9c9, regionState=CLOSED 2024-11-19T15:15:18,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:15:18,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-19T15:15:18,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 in 163 msec 2024-11-19T15:15:18,744 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, REOPEN/MOVE; state=CLOSED, location=50570b5b74e7,46557,1732029218639; forceNewPlan=false, retain=true 2024-11-19T15:15:18,895 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=09d12e268a82f5728962d61ed7bcb9c9, regionState=OPENING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:18,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, REOPEN/MOVE because future has completed 2024-11-19T15:15:18,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:15:19,054 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,054 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} 2024-11-19T15:15:19,054 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,055 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:15:19,055 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,055 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,056 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,057 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:15:19,057 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09d12e268a82f5728962d61ed7bcb9c9 columnFamilyName A 2024-11-19T15:15:19,058 DEBUG [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:19,058 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.HStore(327): Store=09d12e268a82f5728962d61ed7bcb9c9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:15:19,059 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,059 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:15:19,059 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09d12e268a82f5728962d61ed7bcb9c9 columnFamilyName B 2024-11-19T15:15:19,059 DEBUG [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:19,060 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.HStore(327): Store=09d12e268a82f5728962d61ed7bcb9c9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:15:19,060 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,060 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:15:19,060 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09d12e268a82f5728962d61ed7bcb9c9 columnFamilyName C 2024-11-19T15:15:19,060 DEBUG [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:19,061 INFO [StoreOpener-09d12e268a82f5728962d61ed7bcb9c9-1 {}] regionserver.HStore(327): Store=09d12e268a82f5728962d61ed7bcb9c9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:15:19,061 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,061 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,061 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,062 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,063 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,063 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,064 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T15:15:19,065 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,066 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 09d12e268a82f5728962d61ed7bcb9c9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66503319, jitterRate=-0.009023323655128479}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T15:15:19,066 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,067 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 09d12e268a82f5728962d61ed7bcb9c9: Running coprocessor pre-open hook at 1732029319055Writing region info on filesystem at 1732029319055Initializing all the Stores at 1732029319055Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029319055Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029319056 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029319056Cleaning up temporary data from old regions at 1732029319063 (+7 ms)Running coprocessor post-open hooks at 1732029319066 (+3 ms)Region opened successfully at 1732029319067 (+1 ms) 2024-11-19T15:15:19,067 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., pid=100, masterSystemTime=1732029319051 2024-11-19T15:15:19,069 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,069 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,070 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=09d12e268a82f5728962d61ed7bcb9c9, regionState=OPEN, openSeqNum=5, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:15:19,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-19T15:15:19,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 in 173 msec 2024-11-19T15:15:19,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-19T15:15:19,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, REOPEN/MOVE in 499 msec 2024-11-19T15:15:19,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-19T15:15:19,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 505 msec 2024-11-19T15:15:19,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 932 msec 2024-11-19T15:15:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-19T15:15:19,080 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:19,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,081 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:19,081 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:19,082 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:19,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:19,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e27827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:19,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:19,084 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f57d39e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:19,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:19,086 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13fafc37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:19,087 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ee110f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,088 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:19,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:19,089 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d88049, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:19,090 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a39c06b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:19,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:19,092 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:19,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:19,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:19,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18bb605b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:19,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:19,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:19,093 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d789088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:19,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:19,095 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6353db56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:19,096 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:19,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@720b7123, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:19,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:19,097 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65942ffe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:19,099 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7238c53e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:19,100 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:19,100 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:19,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4b5743, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:19,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:19,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:19,101 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49da23ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:19,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:19,103 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5bd3c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:19,104 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:19,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c521114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,108 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:19,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:19,109 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b33e71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:19,110 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc0d5a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d553196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,111 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:19,111 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:19,112 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a0eb6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:19,113 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514a52b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:19,114 DEBUG [Thread-1591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:19,114 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:19,114 DEBUG [Thread-1576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:19,115 DEBUG [Thread-1589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:19,115 DEBUG [Thread-1593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:19,115 DEBUG [Thread-1587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:19,115 DEBUG [Thread-1595 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-19T15:15:19,116 DEBUG [Thread-1593 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:19,116 DEBUG [Thread-1580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:19,116 DEBUG [Thread-1578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:19,116 DEBUG [Thread-1593 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T15:15:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:19,117 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:19,118 DEBUG [Thread-1582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:19,118 DEBUG [Thread-1595 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:19,118 DEBUG [Thread-1595 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:19,119 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:19,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:19,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:19,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:19,115 DEBUG [Thread-1584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:19,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:19,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:19,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:19,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:19,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:19,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:19,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:15:19,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:15:19,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:15:19,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:15:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:15:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:15:19,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:15:19,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:15:19,127 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:19,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:15:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:15:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:19,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:19,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191cbf668d37b04a4bb7080cdb4602437f_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029319125/Put/seqid=0 2024-11-19T15:15:19,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029379172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029319160, max timestamp=1732029319177Num uniques 859; 2024-11-19T15:15:19,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029319160, max timestamp=1732029319177Num uniques 859; 2024-11-19T15:15:19,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:15:19,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:15:19,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029319160, max timestamp=1732029319177Num uniques 859; 2024-11-19T15:15:19,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:15:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742171_1347 (size=14594) 2024-11-19T15:15:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T15:15:19,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029379212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029379221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029379222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029379227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T15:15:19,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:19,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029379284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029379334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029379334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029379344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029379354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T15:15:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T15:15:19,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029379494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029379554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029379554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029379564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029379564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T15:15:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,586 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:19,590 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191cbf668d37b04a4bb7080cdb4602437f_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191cbf668d37b04a4bb7080cdb4602437f_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:19,591 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/ab3ee74bb17b4bf6982d66d5b518ca45, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:19,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/ab3ee74bb17b4bf6982d66d5b518ca45 is 175, key is test_row_0/A:col10/1732029319125/Put/seqid=0 2024-11-19T15:15:19,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742172_1348 (size=39549) 2024-11-19T15:15:19,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T15:15:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T15:15:19,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029379815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T15:15:19,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029379874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:19,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:19,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029379874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:19,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029379884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:19,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029379884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:19,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:19,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:20,006 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/ab3ee74bb17b4bf6982d66d5b518ca45 2024-11-19T15:15:20,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/98e34250be094c8ca81f16a3b94a4018 is 50, key is test_row_0/B:col10/1732029319125/Put/seqid=0 2024-11-19T15:15:20,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T15:15:20,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:20,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742173_1349 (size=12001) 2024-11-19T15:15:20,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T15:15:20,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:20,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T15:15:20,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029380334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:20,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T15:15:20,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:20,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029380394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029380394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:20,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029380394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:20,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029380404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:20,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:20,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/98e34250be094c8ca81f16a3b94a4018 2024-11-19T15:15:20,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/218e0302463c456f937a9f6e5bc7e1c1 is 50, key is test_row_0/C:col10/1732029319125/Put/seqid=0 2024-11-19T15:15:20,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742174_1350 (size=12001) 2024-11-19T15:15:20,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T15:15:20,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:20,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T15:15:20,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:20,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T15:15:20,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:20,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:20,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/218e0302463c456f937a9f6e5bc7e1c1 2024-11-19T15:15:20,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/ab3ee74bb17b4bf6982d66d5b518ca45 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/ab3ee74bb17b4bf6982d66d5b518ca45 2024-11-19T15:15:20,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/ab3ee74bb17b4bf6982d66d5b518ca45, entries=200, sequenceid=15, filesize=38.6 K 2024-11-19T15:15:20,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/98e34250be094c8ca81f16a3b94a4018 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/98e34250be094c8ca81f16a3b94a4018 2024-11-19T15:15:20,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/98e34250be094c8ca81f16a3b94a4018, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T15:15:20,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/218e0302463c456f937a9f6e5bc7e1c1 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/218e0302463c456f937a9f6e5bc7e1c1 2024-11-19T15:15:20,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/218e0302463c456f937a9f6e5bc7e1c1, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T15:15:20,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 09d12e268a82f5728962d61ed7bcb9c9 in 1749ms, sequenceid=15, compaction requested=false 2024-11-19T15:15:20,882 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T15:15:20,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:20,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T15:15:20,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:20,957 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T15:15:20,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:20,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:20,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:20,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:20,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:20,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:20,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a11b15fa78734d27a1e58d203ed5b6ac_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029319226/Put/seqid=0 2024-11-19T15:15:20,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742175_1351 (size=12154) 2024-11-19T15:15:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T15:15:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:21,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:21,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:21,374 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a11b15fa78734d27a1e58d203ed5b6ac_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a11b15fa78734d27a1e58d203ed5b6ac_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:21,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/0a4287c5cf0442cca79415adb36c31ac, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:21,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/0a4287c5cf0442cca79415adb36c31ac is 175, key is test_row_0/A:col10/1732029319226/Put/seqid=0 2024-11-19T15:15:21,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742176_1352 (size=30955) 2024-11-19T15:15:21,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029381414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:21,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029381414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029381414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:21,426 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,426 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:21,426 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:21,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029381418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:21,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029381424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:21,432 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:21,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:21,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029381534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:21,710 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T15:15:21,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029381754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:21,780 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/0a4287c5cf0442cca79415adb36c31ac 2024-11-19T15:15:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/5cc8fb2b54924ef5afe49e89bb2dd6c9 is 50, key is test_row_0/B:col10/1732029319226/Put/seqid=0 2024-11-19T15:15:21,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742177_1353 (size=12001) 2024-11-19T15:15:22,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029382064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:22,193 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/5cc8fb2b54924ef5afe49e89bb2dd6c9 2024-11-19T15:15:22,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/9b30d9cdc70d446488656b46790cf6f9 is 50, key is test_row_0/C:col10/1732029319226/Put/seqid=0 2024-11-19T15:15:22,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742178_1354 (size=12001) 2024-11-19T15:15:22,206 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/9b30d9cdc70d446488656b46790cf6f9 2024-11-19T15:15:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/0a4287c5cf0442cca79415adb36c31ac as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a4287c5cf0442cca79415adb36c31ac 2024-11-19T15:15:22,213 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a4287c5cf0442cca79415adb36c31ac, entries=150, sequenceid=50, filesize=30.2 K 2024-11-19T15:15:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/5cc8fb2b54924ef5afe49e89bb2dd6c9 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5cc8fb2b54924ef5afe49e89bb2dd6c9 2024-11-19T15:15:22,217 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5cc8fb2b54924ef5afe49e89bb2dd6c9, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T15:15:22,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/9b30d9cdc70d446488656b46790cf6f9 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/9b30d9cdc70d446488656b46790cf6f9 2024-11-19T15:15:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,224 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/9b30d9cdc70d446488656b46790cf6f9, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T15:15:22,225 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 09d12e268a82f5728962d61ed7bcb9c9 in 1267ms, sequenceid=50, compaction requested=false 2024-11-19T15:15:22,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:22,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:22,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-19T15:15:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-19T15:15:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-19T15:15:22,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1080 sec 2024-11-19T15:15:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.1150 sec 2024-11-19T15:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T15:15:22,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:22,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:22,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:22,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:22,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f60f1aa99a5645c19c3da49b836d66b4_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_1/A:col10/1732029321416/Put/seqid=0 2024-11-19T15:15:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742179_1355 (size=12154) 2024-11-19T15:15:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:22,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029382804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:22,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029382914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:23,025 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:23,029 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f60f1aa99a5645c19c3da49b836d66b4_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f60f1aa99a5645c19c3da49b836d66b4_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:23,030 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/d58f98254ca6408e8d0e6679b09dc1dd, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:23,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/d58f98254ca6408e8d0e6679b09dc1dd is 175, key is test_row_1/A:col10/1732029321416/Put/seqid=0 2024-11-19T15:15:23,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742180_1356 (size=30951) 2024-11-19T15:15:23,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029383134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T15:15:23,284 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:23,285 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-19T15:15:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T15:15:23,286 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:23,287 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:23,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T15:15:23,435 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/d58f98254ca6408e8d0e6679b09dc1dd 2024-11-19T15:15:23,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029383434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:23,438 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:23,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T15:15:23,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:23,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:23,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:23,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:23,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:23,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/dcb79ed1dbf54266896bc3dd42af2b05 is 50, key is test_row_1/B:col10/1732029321416/Put/seqid=0 2024-11-19T15:15:23,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742181_1357 (size=9657) 2024-11-19T15:15:23,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029383444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:23,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029383445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:23,451 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:23,452 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:23,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/dcb79ed1dbf54266896bc3dd42af2b05 2024-11-19T15:15:23,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/e3d64752990a4809803f594775af0314 is 50, key is test_row_1/C:col10/1732029321416/Put/seqid=0 2024-11-19T15:15:23,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029383454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:23,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029383454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:23,463 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:23,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:23,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:23,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:23,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742182_1358 (size=9657) 2024-11-19T15:15:23,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/e3d64752990a4809803f594775af0314 2024-11-19T15:15:23,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/d58f98254ca6408e8d0e6679b09dc1dd as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/d58f98254ca6408e8d0e6679b09dc1dd 2024-11-19T15:15:23,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/d58f98254ca6408e8d0e6679b09dc1dd, entries=150, sequenceid=62, filesize=30.2 K 2024-11-19T15:15:23,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/dcb79ed1dbf54266896bc3dd42af2b05 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/dcb79ed1dbf54266896bc3dd42af2b05 2024-11-19T15:15:23,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/dcb79ed1dbf54266896bc3dd42af2b05, entries=100, sequenceid=62, filesize=9.4 K 2024-11-19T15:15:23,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/e3d64752990a4809803f594775af0314 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/e3d64752990a4809803f594775af0314 2024-11-19T15:15:23,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/e3d64752990a4809803f594775af0314, entries=100, sequenceid=62, filesize=9.4 K 2024-11-19T15:15:23,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 09d12e268a82f5728962d61ed7bcb9c9 in 890ms, sequenceid=62, compaction requested=true 2024-11-19T15:15:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:23,493 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:23,493 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:23,494 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:23,494 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:23,494 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/B is initiating minor compaction (all files) 2024-11-19T15:15:23,494 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/A is initiating minor compaction (all files) 2024-11-19T15:15:23,494 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/B in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:23,494 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/A in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:23,494 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/98e34250be094c8ca81f16a3b94a4018, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5cc8fb2b54924ef5afe49e89bb2dd6c9, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/dcb79ed1dbf54266896bc3dd42af2b05] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=32.9 K 2024-11-19T15:15:23,494 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/ab3ee74bb17b4bf6982d66d5b518ca45, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a4287c5cf0442cca79415adb36c31ac, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/d58f98254ca6408e8d0e6679b09dc1dd] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=99.1 K 2024-11-19T15:15:23,494 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:23,494 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/ab3ee74bb17b4bf6982d66d5b518ca45, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a4287c5cf0442cca79415adb36c31ac, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/d58f98254ca6408e8d0e6679b09dc1dd] 2024-11-19T15:15:23,494 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 98e34250be094c8ca81f16a3b94a4018, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732029319125 2024-11-19T15:15:23,495 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab3ee74bb17b4bf6982d66d5b518ca45, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732029319125 2024-11-19T15:15:23,495 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cc8fb2b54924ef5afe49e89bb2dd6c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732029319207 2024-11-19T15:15:23,495 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a4287c5cf0442cca79415adb36c31ac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732029319207 2024-11-19T15:15:23,495 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb79ed1dbf54266896bc3dd42af2b05, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732029321416 2024-11-19T15:15:23,495 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting d58f98254ca6408e8d0e6679b09dc1dd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732029321411 2024-11-19T15:15:23,503 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#B#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:23,503 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:23,504 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/e13569a61b20492bb5c3d20b8d9b146e is 50, key is test_row_0/B:col10/1732029319226/Put/seqid=0 2024-11-19T15:15:23,506 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111928febca234e1412aa023918912c6c99f_09d12e268a82f5728962d61ed7bcb9c9 store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:23,508 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111928febca234e1412aa023918912c6c99f_09d12e268a82f5728962d61ed7bcb9c9, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:23,508 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111928febca234e1412aa023918912c6c99f_09d12e268a82f5728962d61ed7bcb9c9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:23,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742183_1359 (size=12104) 2024-11-19T15:15:23,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742184_1360 (size=4469) 2024-11-19T15:15:23,516 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/e13569a61b20492bb5c3d20b8d9b146e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/e13569a61b20492bb5c3d20b8d9b146e 2024-11-19T15:15:23,516 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#A#compaction#309 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:23,517 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/e853432ebc7a4f49be23b752927aa45d is 175, key is test_row_0/A:col10/1732029319226/Put/seqid=0 2024-11-19T15:15:23,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742185_1361 (size=31165) 2024-11-19T15:15:23,527 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/B of 09d12e268a82f5728962d61ed7bcb9c9 into e13569a61b20492bb5c3d20b8d9b146e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:23,527 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:23,527 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/B, priority=13, startTime=1732029323493; duration=0sec 2024-11-19T15:15:23,527 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:23,527 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:B 2024-11-19T15:15:23,528 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:23,528 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:23,529 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/C is initiating minor compaction (all files) 2024-11-19T15:15:23,529 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/C in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:23,529 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/218e0302463c456f937a9f6e5bc7e1c1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/9b30d9cdc70d446488656b46790cf6f9, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/e3d64752990a4809803f594775af0314] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=32.9 K 2024-11-19T15:15:23,529 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 218e0302463c456f937a9f6e5bc7e1c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732029319125 2024-11-19T15:15:23,529 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b30d9cdc70d446488656b46790cf6f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732029319207 2024-11-19T15:15:23,529 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/e853432ebc7a4f49be23b752927aa45d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/e853432ebc7a4f49be23b752927aa45d 2024-11-19T15:15:23,530 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d64752990a4809803f594775af0314, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732029321416 2024-11-19T15:15:23,534 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/A of 09d12e268a82f5728962d61ed7bcb9c9 into e853432ebc7a4f49be23b752927aa45d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:23,534 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:23,534 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/A, priority=13, startTime=1732029323493; duration=0sec 2024-11-19T15:15:23,534 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:23,534 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:A 2024-11-19T15:15:23,536 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#C#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:23,537 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/d872a486746b444e832b14358bc87d90 is 50, key is test_row_0/C:col10/1732029319226/Put/seqid=0 2024-11-19T15:15:23,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742186_1362 (size=12104) 2024-11-19T15:15:23,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T15:15:23,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:23,594 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T15:15:23,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:23,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c6b9a73ff9d34fbb9202159ad3b8288c_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029322763/Put/seqid=0 2024-11-19T15:15:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T15:15:23,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742187_1363 (size=12154) 2024-11-19T15:15:23,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:23,613 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c6b9a73ff9d34fbb9202159ad3b8288c_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c6b9a73ff9d34fbb9202159ad3b8288c_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:23,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/4d8ceaa7fc9c4958803ff9d5578ec2dc, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:23,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/4d8ceaa7fc9c4958803ff9d5578ec2dc is 175, key is test_row_0/A:col10/1732029322763/Put/seqid=0 2024-11-19T15:15:23,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742188_1364 (size=30955) 2024-11-19T15:15:23,620 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/4d8ceaa7fc9c4958803ff9d5578ec2dc 2024-11-19T15:15:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/8e9773580b4f4133b17b6b56dc15f3d5 is 50, key is test_row_0/B:col10/1732029322763/Put/seqid=0 2024-11-19T15:15:23,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742189_1365 (size=12001) 2024-11-19T15:15:23,636 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/8e9773580b4f4133b17b6b56dc15f3d5 2024-11-19T15:15:23,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/48bab867087f454686c54c37f3c2182b is 50, key is test_row_0/C:col10/1732029322763/Put/seqid=0 2024-11-19T15:15:23,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742190_1366 (size=12001) 2024-11-19T15:15:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T15:15:23,951 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/d872a486746b444e832b14358bc87d90 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/d872a486746b444e832b14358bc87d90 2024-11-19T15:15:23,955 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/C of 09d12e268a82f5728962d61ed7bcb9c9 into d872a486746b444e832b14358bc87d90(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:23,955 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:23,955 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/C, priority=13, startTime=1732029323493; duration=0sec 2024-11-19T15:15:23,955 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:23,955 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:C 2024-11-19T15:15:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:23,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:24,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029384029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:24,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:24,048 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/48bab867087f454686c54c37f3c2182b 2024-11-19T15:15:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/4d8ceaa7fc9c4958803ff9d5578ec2dc as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/4d8ceaa7fc9c4958803ff9d5578ec2dc 2024-11-19T15:15:24,056 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/4d8ceaa7fc9c4958803ff9d5578ec2dc, entries=150, sequenceid=84, filesize=30.2 K 2024-11-19T15:15:24,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/8e9773580b4f4133b17b6b56dc15f3d5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8e9773580b4f4133b17b6b56dc15f3d5 2024-11-19T15:15:24,060 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8e9773580b4f4133b17b6b56dc15f3d5, entries=150, sequenceid=84, filesize=11.7 K 2024-11-19T15:15:24,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/48bab867087f454686c54c37f3c2182b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/48bab867087f454686c54c37f3c2182b 2024-11-19T15:15:24,065 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/48bab867087f454686c54c37f3c2182b, entries=150, sequenceid=84, filesize=11.7 K 2024-11-19T15:15:24,066 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 09d12e268a82f5728962d61ed7bcb9c9 in 472ms, sequenceid=84, compaction requested=false 2024-11-19T15:15:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:24,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-19T15:15:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-19T15:15:24,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-19T15:15:24,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 780 msec 2024-11-19T15:15:24,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 784 msec 2024-11-19T15:15:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:24,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T15:15:24,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:24,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:24,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:24,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:24,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:24,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:24,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ec3720cc461a4c508836e37d3648e928_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029324022/Put/seqid=0 2024-11-19T15:15:24,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742191_1367 (size=14594) 2024-11-19T15:15:24,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029384299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:24,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:24,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:24,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:24,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029384404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:24,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:24,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:24,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T15:15:24,424 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:24,424 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-19T15:15:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T15:15:24,426 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:24,427 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:24,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T15:15:24,563 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:24,566 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ec3720cc461a4c508836e37d3648e928_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ec3720cc461a4c508836e37d3648e928_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:24,567 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/da601fe01c69405688706de278a06f58, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:24,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/da601fe01c69405688706de278a06f58 is 175, key is test_row_0/A:col10/1732029324022/Put/seqid=0 2024-11-19T15:15:24,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742192_1368 (size=39549) 2024-11-19T15:15:24,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T15:15:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:24,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:24,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029384624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:24,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T15:15:24,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:24,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:24,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:24,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:24,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T15:15:24,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T15:15:24,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:24,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:24,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:24,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:24,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:24,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029384944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:24,972 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/da601fe01c69405688706de278a06f58 2024-11-19T15:15:24,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/7f048a1c0e9540e893ab788eef63a2d6 is 50, key is test_row_0/B:col10/1732029324022/Put/seqid=0 2024-11-19T15:15:24,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742193_1369 (size=12001) 2024-11-19T15:15:25,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T15:15:25,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:25,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T15:15:25,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T15:15:25,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:25,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T15:15:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/7f048a1c0e9540e893ab788eef63a2d6 2024-11-19T15:15:25,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/f3074f247a6c4a66b13844efd961ca66 is 50, key is test_row_0/C:col10/1732029324022/Put/seqid=0 2024-11-19T15:15:25,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742194_1370 (size=12001) 2024-11-19T15:15:25,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029385464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:25,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T15:15:25,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:25,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T15:15:25,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T15:15:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:25,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/f3074f247a6c4a66b13844efd961ca66 2024-11-19T15:15:25,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/da601fe01c69405688706de278a06f58 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/da601fe01c69405688706de278a06f58 2024-11-19T15:15:25,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T15:15:25,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:25,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:25,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/da601fe01c69405688706de278a06f58, entries=200, sequenceid=98, filesize=38.6 K 2024-11-19T15:15:25,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/7f048a1c0e9540e893ab788eef63a2d6 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/7f048a1c0e9540e893ab788eef63a2d6 2024-11-19T15:15:25,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/7f048a1c0e9540e893ab788eef63a2d6, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T15:15:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/f3074f247a6c4a66b13844efd961ca66 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/f3074f247a6c4a66b13844efd961ca66 2024-11-19T15:15:25,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/f3074f247a6c4a66b13844efd961ca66, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T15:15:25,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 09d12e268a82f5728962d61ed7bcb9c9 in 1664ms, sequenceid=98, compaction requested=true 2024-11-19T15:15:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:25,815 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:25,815 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:25,816 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:25,816 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:25,816 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/B is initiating minor compaction (all files) 2024-11-19T15:15:25,816 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/A is initiating minor compaction (all files) 2024-11-19T15:15:25,816 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/A in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,816 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/B in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,816 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/e13569a61b20492bb5c3d20b8d9b146e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8e9773580b4f4133b17b6b56dc15f3d5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/7f048a1c0e9540e893ab788eef63a2d6] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=35.3 K 2024-11-19T15:15:25,816 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/e853432ebc7a4f49be23b752927aa45d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/4d8ceaa7fc9c4958803ff9d5578ec2dc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/da601fe01c69405688706de278a06f58] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=99.3 K 2024-11-19T15:15:25,817 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,817 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/e853432ebc7a4f49be23b752927aa45d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/4d8ceaa7fc9c4958803ff9d5578ec2dc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/da601fe01c69405688706de278a06f58] 2024-11-19T15:15:25,817 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting e13569a61b20492bb5c3d20b8d9b146e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732029319226 2024-11-19T15:15:25,817 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e853432ebc7a4f49be23b752927aa45d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732029319226 2024-11-19T15:15:25,817 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e9773580b4f4133b17b6b56dc15f3d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732029322763 2024-11-19T15:15:25,817 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d8ceaa7fc9c4958803ff9d5578ec2dc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732029322763 2024-11-19T15:15:25,818 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f048a1c0e9540e893ab788eef63a2d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732029324022 2024-11-19T15:15:25,818 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting da601fe01c69405688706de278a06f58, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732029323986 2024-11-19T15:15:25,823 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:25,826 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:25,826 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/8ff5610dfd134fe3af40b1b8fbaba86d is 50, key is test_row_0/B:col10/1732029324022/Put/seqid=0 2024-11-19T15:15:25,827 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411193140b070eed7422999a53bc91e3ada0c_09d12e268a82f5728962d61ed7bcb9c9 store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:25,829 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411193140b070eed7422999a53bc91e3ada0c_09d12e268a82f5728962d61ed7bcb9c9, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:25,829 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193140b070eed7422999a53bc91e3ada0c_09d12e268a82f5728962d61ed7bcb9c9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:25,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742195_1371 (size=12207) 2024-11-19T15:15:25,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742196_1372 (size=4469) 2024-11-19T15:15:25,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T15:15:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:25,959 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T15:15:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:25,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194b01a52700a3413889a01380b34376c3_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029324236/Put/seqid=0 2024-11-19T15:15:25,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742197_1373 (size=12154) 2024-11-19T15:15:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:25,975 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194b01a52700a3413889a01380b34376c3_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194b01a52700a3413889a01380b34376c3_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:25,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/43da71a3abb84e7a8ef9c333f0c28d3c, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:25,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/43da71a3abb84e7a8ef9c333f0c28d3c is 175, key is test_row_0/A:col10/1732029324236/Put/seqid=0 2024-11-19T15:15:25,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742198_1374 (size=30955) 2024-11-19T15:15:26,235 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#A#compaction#317 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:26,236 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/fa5c28abb95a457ba93272cf69bc57b6 is 175, key is test_row_0/A:col10/1732029324022/Put/seqid=0 2024-11-19T15:15:26,239 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/8ff5610dfd134fe3af40b1b8fbaba86d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8ff5610dfd134fe3af40b1b8fbaba86d 2024-11-19T15:15:26,244 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/B of 09d12e268a82f5728962d61ed7bcb9c9 into 8ff5610dfd134fe3af40b1b8fbaba86d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:26,244 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:26,244 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/B, priority=13, startTime=1732029325815; duration=0sec 2024-11-19T15:15:26,244 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:26,244 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:B 2024-11-19T15:15:26,244 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:26,245 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:26,245 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/C is initiating minor compaction (all files) 2024-11-19T15:15:26,245 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/C in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:26,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742199_1375 (size=31161) 2024-11-19T15:15:26,246 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/d872a486746b444e832b14358bc87d90, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/48bab867087f454686c54c37f3c2182b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/f3074f247a6c4a66b13844efd961ca66] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=35.3 K 2024-11-19T15:15:26,246 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d872a486746b444e832b14358bc87d90, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732029319226 2024-11-19T15:15:26,246 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 48bab867087f454686c54c37f3c2182b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732029322763 2024-11-19T15:15:26,246 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting f3074f247a6c4a66b13844efd961ca66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732029324022 2024-11-19T15:15:26,254 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:26,255 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/9b0e95ba3b1f4a3a87e1e14dd0d0faab is 50, key is test_row_0/C:col10/1732029324022/Put/seqid=0 2024-11-19T15:15:26,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742200_1376 (size=12207) 2024-11-19T15:15:26,267 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/9b0e95ba3b1f4a3a87e1e14dd0d0faab as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/9b0e95ba3b1f4a3a87e1e14dd0d0faab 2024-11-19T15:15:26,274 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/C of 09d12e268a82f5728962d61ed7bcb9c9 into 9b0e95ba3b1f4a3a87e1e14dd0d0faab(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:26,274 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:26,274 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/C, priority=13, startTime=1732029325815; duration=0sec 2024-11-19T15:15:26,274 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:26,274 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:C 2024-11-19T15:15:26,381 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/43da71a3abb84e7a8ef9c333f0c28d3c 2024-11-19T15:15:26,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/3b50f19490fb435b8b46753664ce30ac is 50, key is test_row_0/B:col10/1732029324236/Put/seqid=0 2024-11-19T15:15:26,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742201_1377 (size=12001) 2024-11-19T15:15:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:26,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:26,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029386558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:26,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:26,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:26,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T15:15:26,651 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/fa5c28abb95a457ba93272cf69bc57b6 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/fa5c28abb95a457ba93272cf69bc57b6 2024-11-19T15:15:26,655 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/A of 09d12e268a82f5728962d61ed7bcb9c9 into fa5c28abb95a457ba93272cf69bc57b6(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:26,655 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:26,655 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/A, priority=13, startTime=1732029325815; duration=0sec 2024-11-19T15:15:26,655 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:26,655 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:A 2024-11-19T15:15:26,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029386674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:26,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:26,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:26,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:26,796 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/3b50f19490fb435b8b46753664ce30ac 2024-11-19T15:15:26,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/1c0a117d36344af4bdf11d56f8eba034 is 50, key is test_row_0/C:col10/1732029324236/Put/seqid=0 2024-11-19T15:15:26,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742202_1378 (size=12001) 2024-11-19T15:15:26,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029386884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:26,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,208 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/1c0a117d36344af4bdf11d56f8eba034 2024-11-19T15:15:27,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/43da71a3abb84e7a8ef9c333f0c28d3c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/43da71a3abb84e7a8ef9c333f0c28d3c 2024-11-19T15:15:27,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029387204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,216 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/43da71a3abb84e7a8ef9c333f0c28d3c, entries=150, sequenceid=117, filesize=30.2 K 2024-11-19T15:15:27,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/3b50f19490fb435b8b46753664ce30ac as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/3b50f19490fb435b8b46753664ce30ac 2024-11-19T15:15:27,220 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/3b50f19490fb435b8b46753664ce30ac, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T15:15:27,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/1c0a117d36344af4bdf11d56f8eba034 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/1c0a117d36344af4bdf11d56f8eba034 2024-11-19T15:15:27,225 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/1c0a117d36344af4bdf11d56f8eba034, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T15:15:27,225 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 09d12e268a82f5728962d61ed7bcb9c9 in 1266ms, sequenceid=117, compaction requested=false 2024-11-19T15:15:27,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:27,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:27,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-19T15:15:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-19T15:15:27,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-19T15:15:27,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8000 sec 2024-11-19T15:15:27,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.8050 sec 2024-11-19T15:15:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:27,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T15:15:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:27,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:27,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198c98e8f704d24eb8ac7a7ede8a7c4cfd_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029327455/Put/seqid=0 2024-11-19T15:15:27,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742203_1379 (size=14794) 2024-11-19T15:15:27,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029387506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029387511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029387518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029387519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029387624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029387624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029387634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029387634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029387724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029387834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029387834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029387855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029387855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:27,879 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:27,883 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198c98e8f704d24eb8ac7a7ede8a7c4cfd_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198c98e8f704d24eb8ac7a7ede8a7c4cfd_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:27,884 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/0a8d23ee391343c68d4e46f4424a233d, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:27,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/0a8d23ee391343c68d4e46f4424a233d is 175, key is test_row_0/A:col10/1732029327455/Put/seqid=0 2024-11-19T15:15:27,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742204_1380 (size=39749) 2024-11-19T15:15:28,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029388154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:28,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029388155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:28,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029388164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:28,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029388164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:28,289 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/0a8d23ee391343c68d4e46f4424a233d 2024-11-19T15:15:28,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/e921038098f3427d9301bcc042da2b8e is 50, key is test_row_0/B:col10/1732029327455/Put/seqid=0 2024-11-19T15:15:28,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742205_1381 (size=12151) 2024-11-19T15:15:28,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/e921038098f3427d9301bcc042da2b8e 2024-11-19T15:15:28,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/07ef6cf2f1ec4f8ba2f41216df5ee3d2 is 50, key is test_row_0/C:col10/1732029327455/Put/seqid=0 2024-11-19T15:15:28,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742206_1382 (size=12151) 2024-11-19T15:15:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T15:15:28,594 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:28,595 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-19T15:15:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T15:15:28,596 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:28,597 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:28,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:28,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029388674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:28,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029388674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:28,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029388674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:28,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029388684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T15:15:28,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/07ef6cf2f1ec4f8ba2f41216df5ee3d2 2024-11-19T15:15:28,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/0a8d23ee391343c68d4e46f4424a233d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a8d23ee391343c68d4e46f4424a233d 2024-11-19T15:15:28,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a8d23ee391343c68d4e46f4424a233d, entries=200, sequenceid=134, filesize=38.8 K 2024-11-19T15:15:28,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/e921038098f3427d9301bcc042da2b8e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/e921038098f3427d9301bcc042da2b8e 2024-11-19T15:15:28,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/e921038098f3427d9301bcc042da2b8e, entries=150, sequenceid=134, filesize=11.9 K 2024-11-19T15:15:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/07ef6cf2f1ec4f8ba2f41216df5ee3d2 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/07ef6cf2f1ec4f8ba2f41216df5ee3d2 2024-11-19T15:15:28,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/07ef6cf2f1ec4f8ba2f41216df5ee3d2, entries=150, sequenceid=134, filesize=11.9 K 2024-11-19T15:15:28,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 09d12e268a82f5728962d61ed7bcb9c9 in 1275ms, sequenceid=134, compaction requested=true 2024-11-19T15:15:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:28,732 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:28,732 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:28,733 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:28,733 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:28,733 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/B is initiating minor compaction (all files) 2024-11-19T15:15:28,733 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/A is initiating minor compaction (all files) 2024-11-19T15:15:28,733 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/B in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:28,733 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/A in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:28,733 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8ff5610dfd134fe3af40b1b8fbaba86d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/3b50f19490fb435b8b46753664ce30ac, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/e921038098f3427d9301bcc042da2b8e] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=35.5 K 2024-11-19T15:15:28,733 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/fa5c28abb95a457ba93272cf69bc57b6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/43da71a3abb84e7a8ef9c333f0c28d3c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a8d23ee391343c68d4e46f4424a233d] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=99.5 K 2024-11-19T15:15:28,733 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:28,734 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/fa5c28abb95a457ba93272cf69bc57b6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/43da71a3abb84e7a8ef9c333f0c28d3c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a8d23ee391343c68d4e46f4424a233d] 2024-11-19T15:15:28,734 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ff5610dfd134fe3af40b1b8fbaba86d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732029324022 2024-11-19T15:15:28,734 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa5c28abb95a457ba93272cf69bc57b6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732029324022 2024-11-19T15:15:28,735 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b50f19490fb435b8b46753664ce30ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732029324236 2024-11-19T15:15:28,735 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43da71a3abb84e7a8ef9c333f0c28d3c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732029324236 2024-11-19T15:15:28,735 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting e921038098f3427d9301bcc042da2b8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732029326533 2024-11-19T15:15:28,735 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a8d23ee391343c68d4e46f4424a233d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732029326533 2024-11-19T15:15:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:28,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T15:15:28,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:28,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:28,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:28,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:28,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:28,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:28,749 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:28,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:28,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:28,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:28,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:28,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:28,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:28,751 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#B#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:28,751 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/bf3dc74f6f1245b090c6f65a608232c4 is 50, key is test_row_0/B:col10/1732029327455/Put/seqid=0 2024-11-19T15:15:28,757 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411191f5e41efc98a4c1494c8ab251eb78cd8_09d12e268a82f5728962d61ed7bcb9c9 store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:28,759 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411191f5e41efc98a4c1494c8ab251eb78cd8_09d12e268a82f5728962d61ed7bcb9c9, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:28,760 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191f5e41efc98a4c1494c8ab251eb78cd8_09d12e268a82f5728962d61ed7bcb9c9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:28,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c1fa82859ff04e7bbaf17dfaa98b77a5_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029327510/Put/seqid=0 2024-11-19T15:15:28,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742207_1383 (size=12459) 2024-11-19T15:15:28,778 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/bf3dc74f6f1245b090c6f65a608232c4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/bf3dc74f6f1245b090c6f65a608232c4 2024-11-19T15:15:28,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742208_1384 (size=4469) 2024-11-19T15:15:28,781 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#A#compaction#327 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:28,781 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/c99d6e9142034f938a198a5e0e7f066a is 175, key is test_row_0/A:col10/1732029327455/Put/seqid=0 2024-11-19T15:15:28,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742209_1385 (size=14794) 2024-11-19T15:15:28,793 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/B of 09d12e268a82f5728962d61ed7bcb9c9 into bf3dc74f6f1245b090c6f65a608232c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:28,793 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:28,793 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/B, priority=13, startTime=1732029328732; duration=0sec 2024-11-19T15:15:28,794 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:28,794 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:B 2024-11-19T15:15:28,794 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:28,795 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:28,795 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/C is initiating minor compaction (all files) 2024-11-19T15:15:28,795 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/C in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:28,795 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/9b0e95ba3b1f4a3a87e1e14dd0d0faab, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/1c0a117d36344af4bdf11d56f8eba034, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/07ef6cf2f1ec4f8ba2f41216df5ee3d2] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=35.5 K 2024-11-19T15:15:28,795 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b0e95ba3b1f4a3a87e1e14dd0d0faab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732029324022 2024-11-19T15:15:28,796 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c0a117d36344af4bdf11d56f8eba034, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732029324236 2024-11-19T15:15:28,796 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 07ef6cf2f1ec4f8ba2f41216df5ee3d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732029326533 2024-11-19T15:15:28,806 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#C#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:28,807 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/8d356b17699c45c5bc634d83a2c6264e is 50, key is test_row_0/C:col10/1732029327455/Put/seqid=0 2024-11-19T15:15:28,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742210_1386 (size=31413) 2024-11-19T15:15:28,815 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/c99d6e9142034f938a198a5e0e7f066a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/c99d6e9142034f938a198a5e0e7f066a 2024-11-19T15:15:28,821 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/A of 09d12e268a82f5728962d61ed7bcb9c9 into c99d6e9142034f938a198a5e0e7f066a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:28,821 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:28,821 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/A, priority=13, startTime=1732029328732; duration=0sec 2024-11-19T15:15:28,821 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:28,821 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:A 2024-11-19T15:15:28,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742211_1387 (size=12459) 2024-11-19T15:15:28,830 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/8d356b17699c45c5bc634d83a2c6264e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8d356b17699c45c5bc634d83a2c6264e 2024-11-19T15:15:28,836 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/C of 09d12e268a82f5728962d61ed7bcb9c9 into 8d356b17699c45c5bc634d83a2c6264e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:28,836 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:28,836 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/C, priority=13, startTime=1732029328732; duration=0sec 2024-11-19T15:15:28,836 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:28,836 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:C 2024-11-19T15:15:28,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029388843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:28,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:28,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:28,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:28,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:28,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:28,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:28,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T15:15:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029388964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:29,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:29,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:29,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,191 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:29,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029389184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:29,196 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c1fa82859ff04e7bbaf17dfaa98b77a5_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c1fa82859ff04e7bbaf17dfaa98b77a5_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:29,198 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/8c32a47700174b7896c71103e060c34a, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:29,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/8c32a47700174b7896c71103e060c34a is 175, key is test_row_0/A:col10/1732029327510/Put/seqid=0 2024-11-19T15:15:29,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742212_1388 (size=39749) 2024-11-19T15:15:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T15:15:29,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:29,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:29,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029389504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:29,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:29,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:29,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,616 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/8c32a47700174b7896c71103e060c34a 2024-11-19T15:15:29,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/83300ccef3c8478f8dc8c174e4f5f2fc is 50, key is test_row_0/B:col10/1732029327510/Put/seqid=0 2024-11-19T15:15:29,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742213_1389 (size=12151) 2024-11-19T15:15:29,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:29,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:29,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029389694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:29,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029389694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:29,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029389694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:29,701 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:29,702 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:29,702 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:29,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029389704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:29,718 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:29,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T15:15:29,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:29,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:29,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:29,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:29,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:29,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:30,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029390024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:30,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/83300ccef3c8478f8dc8c174e4f5f2fc 2024-11-19T15:15:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:30,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/8da942bfc6954b9e88932ec352c68785 is 50, key is test_row_0/C:col10/1732029327510/Put/seqid=0 2024-11-19T15:15:30,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742214_1390 (size=12151) 2024-11-19T15:15:30,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:30,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:30,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:30,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:30,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:30,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:30,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:30,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:30,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:30,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:30,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:30,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:30,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:30,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:30,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:30,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:30,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:30,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:30,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/8da942bfc6954b9e88932ec352c68785 2024-11-19T15:15:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:30,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/8c32a47700174b7896c71103e060c34a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/8c32a47700174b7896c71103e060c34a 2024-11-19T15:15:30,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/8c32a47700174b7896c71103e060c34a, entries=200, sequenceid=153, filesize=38.8 K 2024-11-19T15:15:30,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/83300ccef3c8478f8dc8c174e4f5f2fc as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/83300ccef3c8478f8dc8c174e4f5f2fc 2024-11-19T15:15:30,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/83300ccef3c8478f8dc8c174e4f5f2fc, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T15:15:30,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/8da942bfc6954b9e88932ec352c68785 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8da942bfc6954b9e88932ec352c68785 2024-11-19T15:15:30,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8da942bfc6954b9e88932ec352c68785, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T15:15:30,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 09d12e268a82f5728962d61ed7bcb9c9 in 1717ms, sequenceid=153, compaction requested=false 2024-11-19T15:15:30,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:30,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T15:15:30,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:30,592 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T15:15:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:30,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111941e6663af9124a5abcd1cfcf50890ea2_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029328837/Put/seqid=0 2024-11-19T15:15:30,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742215_1391 (size=12304) 2024-11-19T15:15:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T15:15:31,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:31,010 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111941e6663af9124a5abcd1cfcf50890ea2_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111941e6663af9124a5abcd1cfcf50890ea2_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:31,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/384bd57693f141e6907ef9079a916230, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:31,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/384bd57693f141e6907ef9079a916230 is 175, key is test_row_0/A:col10/1732029328837/Put/seqid=0 2024-11-19T15:15:31,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742216_1392 (size=31105) 2024-11-19T15:15:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:31,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:31,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029391165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:31,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029391284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:31,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:31,415 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/384bd57693f141e6907ef9079a916230 2024-11-19T15:15:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/21c02e17167d40f28e1f08b442d421a3 is 50, key is test_row_0/B:col10/1732029328837/Put/seqid=0 2024-11-19T15:15:31,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742217_1393 (size=12151) 2024-11-19T15:15:31,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029391494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:31,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029391724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:31,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029391724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:31,735 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:31,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029391724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:31,735 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_1' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:31,735 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:31,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029391744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:31,749 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 09d12e268a82f5728962d61ed7bcb9c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:31,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029391814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:31,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:31,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:31,827 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/21c02e17167d40f28e1f08b442d421a3 2024-11-19T15:15:31,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/625186935a5242e2b219c41d78883510 is 50, key is test_row_0/C:col10/1732029328837/Put/seqid=0 2024-11-19T15:15:31,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742218_1394 (size=12151) 2024-11-19T15:15:32,238 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/625186935a5242e2b219c41d78883510 2024-11-19T15:15:32,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/384bd57693f141e6907ef9079a916230 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/384bd57693f141e6907ef9079a916230 2024-11-19T15:15:32,245 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/384bd57693f141e6907ef9079a916230, entries=150, sequenceid=169, filesize=30.4 K 2024-11-19T15:15:32,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/21c02e17167d40f28e1f08b442d421a3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/21c02e17167d40f28e1f08b442d421a3 2024-11-19T15:15:32,249 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/21c02e17167d40f28e1f08b442d421a3, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T15:15:32,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/625186935a5242e2b219c41d78883510 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/625186935a5242e2b219c41d78883510 2024-11-19T15:15:32,254 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/625186935a5242e2b219c41d78883510, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T15:15:32,255 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 09d12e268a82f5728962d61ed7bcb9c9 in 1662ms, sequenceid=169, compaction requested=true 2024-11-19T15:15:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-19T15:15:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-19T15:15:32,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-19T15:15:32,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6590 sec 2024-11-19T15:15:32,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.6630 sec 2024-11-19T15:15:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:32,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T15:15:32,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:32,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:32,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:32,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:32,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:32,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:32,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199f4a822684fb40c38967e90441ef713d_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029331132/Put/seqid=0 2024-11-19T15:15:32,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742219_1395 (size=14794) 2024-11-19T15:15:32,353 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:32,357 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199f4a822684fb40c38967e90441ef713d_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199f4a822684fb40c38967e90441ef713d_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:32,357 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/52a0d77d00294df89d2ffb24a97fd150, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:32,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/52a0d77d00294df89d2ffb24a97fd150 is 175, key is test_row_0/A:col10/1732029331132/Put/seqid=0 2024-11-19T15:15:32,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742220_1396 (size=39749) 2024-11-19T15:15:32,362 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/52a0d77d00294df89d2ffb24a97fd150 2024-11-19T15:15:32,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/8c9e2403b8fe41eb89b5d1dadca19313 is 50, key is test_row_0/B:col10/1732029331132/Put/seqid=0 2024-11-19T15:15:32,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742221_1397 (size=12151) 2024-11-19T15:15:32,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029392419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:32,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:32,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:32,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:32,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:32,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029392535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:32,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029392754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:32,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/8c9e2403b8fe41eb89b5d1dadca19313 2024-11-19T15:15:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T15:15:32,774 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:32,774 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-19T15:15:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T15:15:32,776 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:32,777 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:32,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:32,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/ba7317997a1849159b68730690a87aa2 is 50, key is test_row_0/C:col10/1732029331132/Put/seqid=0 2024-11-19T15:15:32,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742222_1398 (size=12151) 2024-11-19T15:15:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T15:15:32,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T15:15:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:32,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:33,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029393064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:33,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:33,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:33,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:33,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T15:15:33,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:33,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:33,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:33,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:33,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T15:15:33,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/ba7317997a1849159b68730690a87aa2 2024-11-19T15:15:33,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/52a0d77d00294df89d2ffb24a97fd150 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/52a0d77d00294df89d2ffb24a97fd150 2024-11-19T15:15:33,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/52a0d77d00294df89d2ffb24a97fd150, entries=200, sequenceid=189, filesize=38.8 K 2024-11-19T15:15:33,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/8c9e2403b8fe41eb89b5d1dadca19313 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8c9e2403b8fe41eb89b5d1dadca19313 2024-11-19T15:15:33,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8c9e2403b8fe41eb89b5d1dadca19313, entries=150, sequenceid=189, filesize=11.9 K 2024-11-19T15:15:33,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/ba7317997a1849159b68730690a87aa2 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/ba7317997a1849159b68730690a87aa2 2024-11-19T15:15:33,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/ba7317997a1849159b68730690a87aa2, entries=150, sequenceid=189, filesize=11.9 K 2024-11-19T15:15:33,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 09d12e268a82f5728962d61ed7bcb9c9 in 859ms, sequenceid=189, compaction requested=true 2024-11-19T15:15:33,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:33,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:33,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:33,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:33,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:33,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:33,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:33,202 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:15:33,202 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:15:33,203 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:15:33,204 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:15:33,204 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/A is initiating minor compaction (all files) 2024-11-19T15:15:33,204 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/B is initiating minor compaction (all files) 2024-11-19T15:15:33,204 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/A in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:33,204 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/B in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:33,204 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/c99d6e9142034f938a198a5e0e7f066a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/8c32a47700174b7896c71103e060c34a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/384bd57693f141e6907ef9079a916230, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/52a0d77d00294df89d2ffb24a97fd150] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=138.7 K 2024-11-19T15:15:33,204 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/bf3dc74f6f1245b090c6f65a608232c4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/83300ccef3c8478f8dc8c174e4f5f2fc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/21c02e17167d40f28e1f08b442d421a3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8c9e2403b8fe41eb89b5d1dadca19313] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=47.8 K 2024-11-19T15:15:33,204 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:33,204 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/c99d6e9142034f938a198a5e0e7f066a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/8c32a47700174b7896c71103e060c34a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/384bd57693f141e6907ef9079a916230, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/52a0d77d00294df89d2ffb24a97fd150] 2024-11-19T15:15:33,204 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting bf3dc74f6f1245b090c6f65a608232c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732029326533 2024-11-19T15:15:33,204 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting c99d6e9142034f938a198a5e0e7f066a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732029326533 2024-11-19T15:15:33,204 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 83300ccef3c8478f8dc8c174e4f5f2fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732029327510 2024-11-19T15:15:33,205 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c32a47700174b7896c71103e060c34a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732029327463 2024-11-19T15:15:33,205 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 21c02e17167d40f28e1f08b442d421a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732029328797 2024-11-19T15:15:33,205 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 384bd57693f141e6907ef9079a916230, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732029328797 2024-11-19T15:15:33,205 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c9e2403b8fe41eb89b5d1dadca19313, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732029331132 2024-11-19T15:15:33,205 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52a0d77d00294df89d2ffb24a97fd150, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732029331132 2024-11-19T15:15:33,214 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:33,216 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119c5b1548992364cd4ba2ac8baa8abbb56_09d12e268a82f5728962d61ed7bcb9c9 store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:33,216 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:33,217 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/355845a723114fe0a7588f7674c62bbf is 50, key is test_row_0/B:col10/1732029331132/Put/seqid=0 2024-11-19T15:15:33,219 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119c5b1548992364cd4ba2ac8baa8abbb56_09d12e268a82f5728962d61ed7bcb9c9, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:33,219 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c5b1548992364cd4ba2ac8baa8abbb56_09d12e268a82f5728962d61ed7bcb9c9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:33,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742223_1399 (size=12595) 2024-11-19T15:15:33,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742224_1400 (size=4469) 2024-11-19T15:15:33,232 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/355845a723114fe0a7588f7674c62bbf as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/355845a723114fe0a7588f7674c62bbf 2024-11-19T15:15:33,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T15:15:33,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:33,235 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T15:15:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:33,248 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/B of 09d12e268a82f5728962d61ed7bcb9c9 into 355845a723114fe0a7588f7674c62bbf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:33,248 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:33,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ac43eb59d9624282969798c2e6c02494_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029332390/Put/seqid=0 2024-11-19T15:15:33,248 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/B, priority=12, startTime=1732029333202; duration=0sec 2024-11-19T15:15:33,248 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:33,249 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:B 2024-11-19T15:15:33,249 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:15:33,250 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:15:33,250 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/C is initiating minor compaction (all files) 2024-11-19T15:15:33,250 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/C in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:33,251 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8d356b17699c45c5bc634d83a2c6264e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8da942bfc6954b9e88932ec352c68785, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/625186935a5242e2b219c41d78883510, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/ba7317997a1849159b68730690a87aa2] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=47.8 K 2024-11-19T15:15:33,251 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d356b17699c45c5bc634d83a2c6264e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732029326533 2024-11-19T15:15:33,251 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8da942bfc6954b9e88932ec352c68785, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732029327510 2024-11-19T15:15:33,252 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 625186935a5242e2b219c41d78883510, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732029328797 2024-11-19T15:15:33,252 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting ba7317997a1849159b68730690a87aa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732029331132 2024-11-19T15:15:33,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742225_1401 (size=12304) 2024-11-19T15:15:33,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:33,260 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ac43eb59d9624282969798c2e6c02494_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac43eb59d9624282969798c2e6c02494_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:33,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/203cc04ae08046b2b2f5500a6246d9b2, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:33,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/203cc04ae08046b2b2f5500a6246d9b2 is 175, key is test_row_0/A:col10/1732029332390/Put/seqid=0 2024-11-19T15:15:33,262 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#C#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:33,263 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/0332d52d4b2041c3a0f955ca1a251c1d is 50, key is test_row_0/C:col10/1732029331132/Put/seqid=0 2024-11-19T15:15:33,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742227_1403 (size=31105) 2024-11-19T15:15:33,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742226_1402 (size=12595) 2024-11-19T15:15:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T15:15:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:33,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:33,628 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#A#compaction#338 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:33,629 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/296ef868ab3f46b79321036468882265 is 175, key is test_row_0/A:col10/1732029331132/Put/seqid=0 2024-11-19T15:15:33,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742228_1404 (size=31549) 2024-11-19T15:15:33,670 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/203cc04ae08046b2b2f5500a6246d9b2 2024-11-19T15:15:33,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/5b278f3271384a2d8d790203b87153de is 50, key is test_row_0/B:col10/1732029332390/Put/seqid=0 2024-11-19T15:15:33,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742229_1405 (size=12151) 2024-11-19T15:15:33,681 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/5b278f3271384a2d8d790203b87153de 2024-11-19T15:15:33,683 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/0332d52d4b2041c3a0f955ca1a251c1d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/0332d52d4b2041c3a0f955ca1a251c1d 2024-11-19T15:15:33,688 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/C of 09d12e268a82f5728962d61ed7bcb9c9 into 0332d52d4b2041c3a0f955ca1a251c1d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:33,688 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:33,688 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/C, priority=12, startTime=1732029333202; duration=0sec 2024-11-19T15:15:33,688 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:33,688 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:C 2024-11-19T15:15:33,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/607d5575c4d84aa6b3566b304160719d is 50, key is test_row_0/C:col10/1732029332390/Put/seqid=0 2024-11-19T15:15:33,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742230_1406 (size=12151) 2024-11-19T15:15:33,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029393754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:33,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029393874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T15:15:34,037 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/296ef868ab3f46b79321036468882265 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/296ef868ab3f46b79321036468882265 2024-11-19T15:15:34,042 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/A of 09d12e268a82f5728962d61ed7bcb9c9 into 296ef868ab3f46b79321036468882265(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:34,042 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:34,042 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/A, priority=12, startTime=1732029333202; duration=0sec 2024-11-19T15:15:34,042 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:34,042 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:A 2024-11-19T15:15:34,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029394084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:34,094 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/607d5575c4d84aa6b3566b304160719d 2024-11-19T15:15:34,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/203cc04ae08046b2b2f5500a6246d9b2 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/203cc04ae08046b2b2f5500a6246d9b2 2024-11-19T15:15:34,105 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/203cc04ae08046b2b2f5500a6246d9b2, entries=150, sequenceid=201, filesize=30.4 K 2024-11-19T15:15:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/5b278f3271384a2d8d790203b87153de as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5b278f3271384a2d8d790203b87153de 2024-11-19T15:15:34,109 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5b278f3271384a2d8d790203b87153de, entries=150, sequenceid=201, filesize=11.9 K 2024-11-19T15:15:34,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/607d5575c4d84aa6b3566b304160719d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/607d5575c4d84aa6b3566b304160719d 2024-11-19T15:15:34,114 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/607d5575c4d84aa6b3566b304160719d, entries=150, sequenceid=201, filesize=11.9 K 2024-11-19T15:15:34,115 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 09d12e268a82f5728962d61ed7bcb9c9 in 880ms, sequenceid=201, compaction requested=false 2024-11-19T15:15:34,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:34,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:34,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-19T15:15:34,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-19T15:15:34,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-19T15:15:34,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3400 sec 2024-11-19T15:15:34,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.3460 sec 2024-11-19T15:15:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:34,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T15:15:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:34,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199dbfbddb1e2840218da09f6eccd6d8fd_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029333680/Put/seqid=0 2024-11-19T15:15:34,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742231_1407 (size=14794) 2024-11-19T15:15:34,425 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:34,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199dbfbddb1e2840218da09f6eccd6d8fd_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199dbfbddb1e2840218da09f6eccd6d8fd_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:34,429 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/babd566ebf87478db9a6fe84e6ad5dd8, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:34,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/babd566ebf87478db9a6fe84e6ad5dd8 is 175, key is test_row_0/A:col10/1732029333680/Put/seqid=0 2024-11-19T15:15:34,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742232_1408 (size=39749) 2024-11-19T15:15:34,435 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/babd566ebf87478db9a6fe84e6ad5dd8 2024-11-19T15:15:34,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/5b1f31a651564940913464ef826a8078 is 50, key is test_row_0/B:col10/1732029333680/Put/seqid=0 2024-11-19T15:15:34,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742233_1409 (size=12151) 2024-11-19T15:15:34,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029394477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:34,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:34,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:34,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:34,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029394594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:34,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029394804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:34,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/5b1f31a651564940913464ef826a8078 2024-11-19T15:15:34,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/752770aa230d495c9a99ab268ef4119a is 50, key is test_row_0/C:col10/1732029333680/Put/seqid=0 2024-11-19T15:15:34,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742234_1410 (size=12151) 2024-11-19T15:15:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T15:15:34,924 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:34,924 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-19T15:15:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T15:15:34,926 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:34,927 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:34,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T15:15:35,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T15:15:35,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:35,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:35,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:35,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:35,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:35,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029395114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:35,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T15:15:35,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:35,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:35,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:35,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:35,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T15:15:35,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/752770aa230d495c9a99ab268ef4119a 2024-11-19T15:15:35,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/babd566ebf87478db9a6fe84e6ad5dd8 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/babd566ebf87478db9a6fe84e6ad5dd8 2024-11-19T15:15:35,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/babd566ebf87478db9a6fe84e6ad5dd8, entries=200, sequenceid=225, filesize=38.8 K 2024-11-19T15:15:35,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/5b1f31a651564940913464ef826a8078 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5b1f31a651564940913464ef826a8078 2024-11-19T15:15:35,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5b1f31a651564940913464ef826a8078, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T15:15:35,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/752770aa230d495c9a99ab268ef4119a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/752770aa230d495c9a99ab268ef4119a 2024-11-19T15:15:35,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/752770aa230d495c9a99ab268ef4119a, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T15:15:35,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 09d12e268a82f5728962d61ed7bcb9c9 in 865ms, sequenceid=225, compaction requested=true 2024-11-19T15:15:35,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:35,276 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:35,276 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:35,277 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:35,277 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:35,277 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/B is initiating minor compaction (all files) 2024-11-19T15:15:35,277 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/A is initiating minor compaction (all files) 2024-11-19T15:15:35,277 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/B in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:35,277 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/A in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:35,277 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/355845a723114fe0a7588f7674c62bbf, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5b278f3271384a2d8d790203b87153de, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5b1f31a651564940913464ef826a8078] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=36.0 K 2024-11-19T15:15:35,277 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/296ef868ab3f46b79321036468882265, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/203cc04ae08046b2b2f5500a6246d9b2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/babd566ebf87478db9a6fe84e6ad5dd8] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=100.0 K 2024-11-19T15:15:35,277 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:35,277 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/296ef868ab3f46b79321036468882265, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/203cc04ae08046b2b2f5500a6246d9b2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/babd566ebf87478db9a6fe84e6ad5dd8] 2024-11-19T15:15:35,278 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 355845a723114fe0a7588f7674c62bbf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732029331132 2024-11-19T15:15:35,278 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 296ef868ab3f46b79321036468882265, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732029331132 2024-11-19T15:15:35,278 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b278f3271384a2d8d790203b87153de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732029332390 2024-11-19T15:15:35,279 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203cc04ae08046b2b2f5500a6246d9b2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732029332390 2024-11-19T15:15:35,279 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b1f31a651564940913464ef826a8078, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732029333680 2024-11-19T15:15:35,279 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting babd566ebf87478db9a6fe84e6ad5dd8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732029333680 2024-11-19T15:15:35,289 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:35,290 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#B#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:35,291 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/d14cdaec8ad743329df621952834e19e is 50, key is test_row_0/B:col10/1732029333680/Put/seqid=0 2024-11-19T15:15:35,294 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119280c3003334a46c4844f9b68d1f681b5_09d12e268a82f5728962d61ed7bcb9c9 store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:35,296 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119280c3003334a46c4844f9b68d1f681b5_09d12e268a82f5728962d61ed7bcb9c9, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:35,296 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119280c3003334a46c4844f9b68d1f681b5_09d12e268a82f5728962d61ed7bcb9c9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:35,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742235_1411 (size=12697) 2024-11-19T15:15:35,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742236_1412 (size=4469) 2024-11-19T15:15:35,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T15:15:35,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:35,386 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T15:15:35,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:35,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:35,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:35,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:35,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:35,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:35,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199ed84e1974784146b13b7780a9b357fb_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029334476/Put/seqid=0 2024-11-19T15:15:35,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742237_1413 (size=12304) 2024-11-19T15:15:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T15:15:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:35,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:35,709 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/d14cdaec8ad743329df621952834e19e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/d14cdaec8ad743329df621952834e19e 2024-11-19T15:15:35,712 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#A#compaction#348 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:35,713 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/fc98e14b8f734687871f41d030745575 is 175, key is test_row_0/A:col10/1732029333680/Put/seqid=0 2024-11-19T15:15:35,717 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/B of 09d12e268a82f5728962d61ed7bcb9c9 into d14cdaec8ad743329df621952834e19e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:35,717 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:35,717 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/B, priority=13, startTime=1732029335276; duration=0sec 2024-11-19T15:15:35,717 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:35,717 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:B 2024-11-19T15:15:35,717 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:35,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742238_1414 (size=31651) 2024-11-19T15:15:35,720 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:35,720 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/C is initiating minor compaction (all files) 2024-11-19T15:15:35,720 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/C in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:35,720 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/0332d52d4b2041c3a0f955ca1a251c1d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/607d5575c4d84aa6b3566b304160719d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/752770aa230d495c9a99ab268ef4119a] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=36.0 K 2024-11-19T15:15:35,720 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 0332d52d4b2041c3a0f955ca1a251c1d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732029331132 2024-11-19T15:15:35,720 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 607d5575c4d84aa6b3566b304160719d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732029332390 2024-11-19T15:15:35,721 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 752770aa230d495c9a99ab268ef4119a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732029333680 2024-11-19T15:15:35,725 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/fc98e14b8f734687871f41d030745575 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/fc98e14b8f734687871f41d030745575 2024-11-19T15:15:35,728 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:35,729 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/747d100d6de64ecfb80f152b239e7d1d is 50, key is test_row_0/C:col10/1732029333680/Put/seqid=0 2024-11-19T15:15:35,730 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/A of 09d12e268a82f5728962d61ed7bcb9c9 into fc98e14b8f734687871f41d030745575(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:35,730 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:35,730 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/A, priority=13, startTime=1732029335276; duration=0sec 2024-11-19T15:15:35,730 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:35,730 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:A 2024-11-19T15:15:35,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742239_1415 (size=12697) 2024-11-19T15:15:35,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732029335737, max timestamp=1732029335768Num uniques 859; 2024-11-19T15:15:35,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732029335737, max timestamp=1732029335768Num uniques 859; 2024-11-19T15:15:35,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T15:15:35,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T15:15:35,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732029335737, max timestamp=1732029335768Num uniques 859; 2024-11-19T15:15:35,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T15:15:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:35,802 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199ed84e1974784146b13b7780a9b357fb_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199ed84e1974784146b13b7780a9b357fb_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:35,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/46cd2cb5c1184ddcb3e70d8a9ede167d, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:35,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/46cd2cb5c1184ddcb3e70d8a9ede167d is 175, key is test_row_0/A:col10/1732029334476/Put/seqid=0 2024-11-19T15:15:35,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742240_1416 (size=31105) 2024-11-19T15:15:35,807 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/46cd2cb5c1184ddcb3e70d8a9ede167d 2024-11-19T15:15:35,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/847f429c199040869700fd7e32f45760 is 50, key is test_row_0/B:col10/1732029334476/Put/seqid=0 2024-11-19T15:15:35,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742241_1417 (size=12151) 2024-11-19T15:15:35,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029395805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:35,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029395807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:35,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:35,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029395808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:35,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:35,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:35,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029395810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:35,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029395811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:35,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029395924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:35,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029395924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:35,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029395935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:35,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029395935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:35,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029395935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:35,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T15:15:36,143 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/747d100d6de64ecfb80f152b239e7d1d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/747d100d6de64ecfb80f152b239e7d1d 2024-11-19T15:15:36,147 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/C of 09d12e268a82f5728962d61ed7bcb9c9 into 747d100d6de64ecfb80f152b239e7d1d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:36,147 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:36,147 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/C, priority=13, startTime=1732029335276; duration=0sec 2024-11-19T15:15:36,148 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:36,148 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:C 2024-11-19T15:15:36,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029396134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029396135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029396154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029396154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029396154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,220 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/847f429c199040869700fd7e32f45760 2024-11-19T15:15:36,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/0b9a8b66357a498ba4af78413e75bd43 is 50, key is test_row_0/C:col10/1732029334476/Put/seqid=0 2024-11-19T15:15:36,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742242_1418 (size=12151) 2024-11-19T15:15:36,234 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/0b9a8b66357a498ba4af78413e75bd43 2024-11-19T15:15:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/46cd2cb5c1184ddcb3e70d8a9ede167d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/46cd2cb5c1184ddcb3e70d8a9ede167d 2024-11-19T15:15:36,241 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/46cd2cb5c1184ddcb3e70d8a9ede167d, entries=150, sequenceid=236, filesize=30.4 K 2024-11-19T15:15:36,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/847f429c199040869700fd7e32f45760 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/847f429c199040869700fd7e32f45760 2024-11-19T15:15:36,245 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/847f429c199040869700fd7e32f45760, entries=150, sequenceid=236, filesize=11.9 K 2024-11-19T15:15:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/0b9a8b66357a498ba4af78413e75bd43 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/0b9a8b66357a498ba4af78413e75bd43 2024-11-19T15:15:36,251 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/0b9a8b66357a498ba4af78413e75bd43, entries=150, sequenceid=236, filesize=11.9 K 2024-11-19T15:15:36,252 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 09d12e268a82f5728962d61ed7bcb9c9 in 866ms, sequenceid=236, compaction requested=false 2024-11-19T15:15:36,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:36,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:36,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-19T15:15:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-19T15:15:36,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-19T15:15:36,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3260 sec 2024-11-19T15:15:36,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.3310 sec 2024-11-19T15:15:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:36,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-19T15:15:36,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:36,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:36,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:36,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:36,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:36,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:36,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195886914a292d4495add01bd95f74ef2d_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029335804/Put/seqid=0 2024-11-19T15:15:36,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742243_1419 (size=14994) 2024-11-19T15:15:36,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029396485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029396486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029396487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029396497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029396497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029396604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029396604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029396604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029396604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029396604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029396824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029396824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029396824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029396824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029396825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:36,875 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:36,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195886914a292d4495add01bd95f74ef2d_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195886914a292d4495add01bd95f74ef2d_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:36,879 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/b2e3dcdfec9e458aa145ecb9e8dede43, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:36,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/b2e3dcdfec9e458aa145ecb9e8dede43 is 175, key is test_row_0/A:col10/1732029335804/Put/seqid=0 2024-11-19T15:15:36,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742244_1420 (size=39949) 2024-11-19T15:15:36,979 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T15:15:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T15:15:37,075 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:37,075 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-19T15:15:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T15:15:37,077 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:37,078 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:37,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:37,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029397134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:37,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029397134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:37,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029397134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029397134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029397134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T15:15:37,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T15:15:37,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:37,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:37,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:37,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,284 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/b2e3dcdfec9e458aa145ecb9e8dede43 2024-11-19T15:15:37,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/3f75b3b404704df8a2957bac076f1201 is 50, key is test_row_0/B:col10/1732029335804/Put/seqid=0 2024-11-19T15:15:37,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742245_1421 (size=12301) 2024-11-19T15:15:37,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T15:15:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:37,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:37,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:37,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T15:15:37,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T15:15:37,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:37,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:37,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:37,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029397654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:37,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029397655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:37,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029397655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029397655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:37,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029397655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:37,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:37,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:37,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T15:15:37,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:37,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:37,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:37,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/3f75b3b404704df8a2957bac076f1201 2024-11-19T15:15:37,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/8f34fda3328e48fdb4f5f8aad94acce3 is 50, key is test_row_0/C:col10/1732029335804/Put/seqid=0 2024-11-19T15:15:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T15:15:37,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742246_1422 (size=12301) 2024-11-19T15:15:37,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T15:15:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:37,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T15:15:37,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:37,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. as already flushing 2024-11-19T15:15:37,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:37,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:38,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/8f34fda3328e48fdb4f5f8aad94acce3 2024-11-19T15:15:38,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/b2e3dcdfec9e458aa145ecb9e8dede43 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/b2e3dcdfec9e458aa145ecb9e8dede43 2024-11-19T15:15:38,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/b2e3dcdfec9e458aa145ecb9e8dede43, entries=200, sequenceid=274, filesize=39.0 K 2024-11-19T15:15:38,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/3f75b3b404704df8a2957bac076f1201 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/3f75b3b404704df8a2957bac076f1201 2024-11-19T15:15:38,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/3f75b3b404704df8a2957bac076f1201, entries=150, sequenceid=274, filesize=12.0 K 2024-11-19T15:15:38,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/8f34fda3328e48fdb4f5f8aad94acce3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8f34fda3328e48fdb4f5f8aad94acce3 2024-11-19T15:15:38,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8f34fda3328e48fdb4f5f8aad94acce3, entries=150, sequenceid=274, filesize=12.0 K 2024-11-19T15:15:38,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 09d12e268a82f5728962d61ed7bcb9c9 in 1661ms, sequenceid=274, compaction requested=true 2024-11-19T15:15:38,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:38,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:38,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:38,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:38,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:38,124 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:38,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:38,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:38,124 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:38,127 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:38,127 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:38,127 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/B is initiating minor compaction (all files) 2024-11-19T15:15:38,127 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/A is initiating minor compaction (all files) 2024-11-19T15:15:38,127 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/B in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:38,127 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/A in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:38,127 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/d14cdaec8ad743329df621952834e19e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/847f429c199040869700fd7e32f45760, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/3f75b3b404704df8a2957bac076f1201] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=36.3 K 2024-11-19T15:15:38,127 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/fc98e14b8f734687871f41d030745575, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/46cd2cb5c1184ddcb3e70d8a9ede167d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/b2e3dcdfec9e458aa145ecb9e8dede43] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=100.3 K 2024-11-19T15:15:38,127 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:38,127 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/fc98e14b8f734687871f41d030745575, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/46cd2cb5c1184ddcb3e70d8a9ede167d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/b2e3dcdfec9e458aa145ecb9e8dede43] 2024-11-19T15:15:38,127 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d14cdaec8ad743329df621952834e19e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732029333680 2024-11-19T15:15:38,127 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc98e14b8f734687871f41d030745575, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732029333680 2024-11-19T15:15:38,127 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 847f429c199040869700fd7e32f45760, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732029334440 2024-11-19T15:15:38,128 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46cd2cb5c1184ddcb3e70d8a9ede167d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732029334440 2024-11-19T15:15:38,128 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f75b3b404704df8a2957bac076f1201, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732029335804 2024-11-19T15:15:38,128 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2e3dcdfec9e458aa145ecb9e8dede43, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732029335804 2024-11-19T15:15:38,139 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#B#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:38,140 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/cf52d8b4d1e44d1bba7a3a023828cc53 is 50, key is test_row_0/B:col10/1732029335804/Put/seqid=0 2024-11-19T15:15:38,141 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:38,145 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119f95ee4421328409dbc6b625a16f74729_09d12e268a82f5728962d61ed7bcb9c9 store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:38,147 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119f95ee4421328409dbc6b625a16f74729_09d12e268a82f5728962d61ed7bcb9c9, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:38,147 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f95ee4421328409dbc6b625a16f74729_09d12e268a82f5728962d61ed7bcb9c9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:38,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T15:15:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:38,151 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T15:15:38,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742247_1423 (size=12949) 2024-11-19T15:15:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:38,156 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/cf52d8b4d1e44d1bba7a3a023828cc53 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/cf52d8b4d1e44d1bba7a3a023828cc53 2024-11-19T15:15:38,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742248_1424 (size=4469) 2024-11-19T15:15:38,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190b719770134d4bd9928624129f2a1d27_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029336496/Put/seqid=0 2024-11-19T15:15:38,162 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#A#compaction#357 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:38,163 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/1290a07916964ff7b4d2901894b27925 is 175, key is test_row_0/A:col10/1732029335804/Put/seqid=0 2024-11-19T15:15:38,165 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/B of 09d12e268a82f5728962d61ed7bcb9c9 into cf52d8b4d1e44d1bba7a3a023828cc53(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:38,165 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:38,165 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/B, priority=13, startTime=1732029338124; duration=0sec 2024-11-19T15:15:38,165 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:38,166 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:B 2024-11-19T15:15:38,166 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:38,166 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:38,166 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/C is initiating minor compaction (all files) 2024-11-19T15:15:38,166 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/C in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:38,167 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/747d100d6de64ecfb80f152b239e7d1d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/0b9a8b66357a498ba4af78413e75bd43, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8f34fda3328e48fdb4f5f8aad94acce3] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=36.3 K 2024-11-19T15:15:38,167 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 747d100d6de64ecfb80f152b239e7d1d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732029333680 2024-11-19T15:15:38,167 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b9a8b66357a498ba4af78413e75bd43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732029334440 2024-11-19T15:15:38,168 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f34fda3328e48fdb4f5f8aad94acce3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732029335804 2024-11-19T15:15:38,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742250_1426 (size=31903) 2024-11-19T15:15:38,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742249_1425 (size=12454) 2024-11-19T15:15:38,184 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:38,185 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/da82736052c645f0ac998033ee8eb83b is 50, key is test_row_0/C:col10/1732029335804/Put/seqid=0 2024-11-19T15:15:38,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,190 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190b719770134d4bd9928624129f2a1d27_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190b719770134d4bd9928624129f2a1d27_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:38,190 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/1290a07916964ff7b4d2901894b27925 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/1290a07916964ff7b4d2901894b27925 2024-11-19T15:15:38,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/792cc4f1d9b2486b9ff31ef928cd3de3, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:38,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/792cc4f1d9b2486b9ff31ef928cd3de3 is 175, key is test_row_0/A:col10/1732029336496/Put/seqid=0 2024-11-19T15:15:38,197 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/A of 09d12e268a82f5728962d61ed7bcb9c9 into 1290a07916964ff7b4d2901894b27925(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:38,197 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:38,197 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/A, priority=13, startTime=1732029338124; duration=0sec 2024-11-19T15:15:38,198 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:38,198 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:A 2024-11-19T15:15:38,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742251_1427 (size=12949) 2024-11-19T15:15:38,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742252_1428 (size=31255) 2024-11-19T15:15:38,203 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/792cc4f1d9b2486b9ff31ef928cd3de3 2024-11-19T15:15:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T15:15:38,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/c944fa5f59b44a3d81dc43dfba546f25 is 50, key is test_row_0/B:col10/1732029336496/Put/seqid=0 2024-11-19T15:15:38,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742253_1429 (size=12301) 2024-11-19T15:15:38,221 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/c944fa5f59b44a3d81dc43dfba546f25 2024-11-19T15:15:38,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/e8d1001325344a6aaaf76d431628fb5d is 50, key is test_row_0/C:col10/1732029336496/Put/seqid=0 2024-11-19T15:15:38,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742254_1430 (size=12301) 2024-11-19T15:15:38,602 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/da82736052c645f0ac998033ee8eb83b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/da82736052c645f0ac998033ee8eb83b 2024-11-19T15:15:38,606 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/C of 09d12e268a82f5728962d61ed7bcb9c9 into da82736052c645f0ac998033ee8eb83b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:38,606 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:38,606 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/C, priority=13, startTime=1732029338124; duration=0sec 2024-11-19T15:15:38,606 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:38,606 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:C 2024-11-19T15:15:38,634 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/e8d1001325344a6aaaf76d431628fb5d 2024-11-19T15:15:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/792cc4f1d9b2486b9ff31ef928cd3de3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/792cc4f1d9b2486b9ff31ef928cd3de3 2024-11-19T15:15:38,645 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/792cc4f1d9b2486b9ff31ef928cd3de3, entries=150, sequenceid=285, filesize=30.5 K 2024-11-19T15:15:38,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/c944fa5f59b44a3d81dc43dfba546f25 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/c944fa5f59b44a3d81dc43dfba546f25 2024-11-19T15:15:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,650 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/c944fa5f59b44a3d81dc43dfba546f25, entries=150, sequenceid=285, filesize=12.0 K 2024-11-19T15:15:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/e8d1001325344a6aaaf76d431628fb5d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/e8d1001325344a6aaaf76d431628fb5d 2024-11-19T15:15:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,656 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/e8d1001325344a6aaaf76d431628fb5d, entries=150, sequenceid=285, filesize=12.0 K 2024-11-19T15:15:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,657 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 09d12e268a82f5728962d61ed7bcb9c9 in 506ms, sequenceid=285, compaction requested=false 2024-11-19T15:15:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:38,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:38,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-19T15:15:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-19T15:15:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-19T15:15:38,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5800 sec 2024-11-19T15:15:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.5860 sec 2024-11-19T15:15:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:38,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:15:38,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:38,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:38,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:38,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:38,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:38,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195814fc2a9edd48268cef1c62d8dd6ada_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029338773/Put/seqid=0 2024-11-19T15:15:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742255_1431 (size=22618) 2024-11-19T15:15:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:38,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732029338806, max timestamp=1732029338835Num uniques 859; 2024-11-19T15:15:38,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732029338806, max timestamp=1732029338835Num uniques 859; 2024-11-19T15:15:38,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732029338806, max timestamp=1732029338835Num uniques 859; 2024-11-19T15:15:38,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T15:15:38,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T15:15:38,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T15:15:38,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029398863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029398867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:38,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029398874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:38,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029398875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:38,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029398876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:38,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029398984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:38,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029398984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:38,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029398985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029398994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:39,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029398994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,125 INFO [Thread-1595 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1595. 2024-11-19T15:15:39,125 DEBUG [Thread-1595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:39,126 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:39,126 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:39,126 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:39,132 INFO [Thread-1589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1589. 2024-11-19T15:15:39,132 INFO [Thread-1591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1591. 2024-11-19T15:15:39,132 DEBUG [Thread-1591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:39,132 DEBUG [Thread-1589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:39,132 DEBUG [Thread-1591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:39,132 DEBUG [Thread-1589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:39,132 DEBUG [Thread-1591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:39,132 DEBUG [Thread-1589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:39,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:39,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:39,139 INFO [Thread-1593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1593. 2024-11-19T15:15:39,139 DEBUG [Thread-1593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:39,139 DEBUG [Thread-1593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:39,139 DEBUG [Thread-1593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:39,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:39,143 INFO [Thread-1587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1587. 2024-11-19T15:15:39,143 DEBUG [Thread-1587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:39,143 DEBUG [Thread-1587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:39,143 DEBUG [Thread-1587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:39,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:39,189 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:39,192 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195814fc2a9edd48268cef1c62d8dd6ada_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195814fc2a9edd48268cef1c62d8dd6ada_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:39,193 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/6c5e4b6bd6b74fcab811999c9698f53b, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:39,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/6c5e4b6bd6b74fcab811999c9698f53b is 175, key is test_row_0/A:col10/1732029338773/Put/seqid=0 2024-11-19T15:15:39,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742256_1432 (size=66023) 2024-11-19T15:15:39,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:39,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:39,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029399205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029399205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029399205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:39,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029399205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029399214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:39,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T15:15:39,225 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029399514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029399515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029399515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029399515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029399524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:39,597 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/6c5e4b6bd6b74fcab811999c9698f53b 2024-11-19T15:15:39,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/8a0485be8a0e4b3dbe366260daa0580b is 50, key is test_row_0/B:col10/1732029338773/Put/seqid=0 2024-11-19T15:15:39,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742257_1433 (size=12301) 2024-11-19T15:15:40,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/8a0485be8a0e4b3dbe366260daa0580b 2024-11-19T15:15:40,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/7d0007d5193c4e69b98000b3190e7909 is 50, key is test_row_0/C:col10/1732029338773/Put/seqid=0 2024-11-19T15:15:40,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742258_1434 (size=12301) 2024-11-19T15:15:40,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:40,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:40,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36118 deadline: 1732029400024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36132 deadline: 1732029400024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36148 deadline: 1732029400024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:40,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36108 deadline: 1732029400024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:40,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36092 deadline: 1732029400034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:15:40,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/7d0007d5193c4e69b98000b3190e7909 2024-11-19T15:15:40,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/6c5e4b6bd6b74fcab811999c9698f53b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/6c5e4b6bd6b74fcab811999c9698f53b 2024-11-19T15:15:40,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/6c5e4b6bd6b74fcab811999c9698f53b, entries=350, sequenceid=298, filesize=64.5 K 2024-11-19T15:15:40,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/8a0485be8a0e4b3dbe366260daa0580b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8a0485be8a0e4b3dbe366260daa0580b 2024-11-19T15:15:40,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8a0485be8a0e4b3dbe366260daa0580b, entries=150, sequenceid=298, filesize=12.0 K 2024-11-19T15:15:40,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/7d0007d5193c4e69b98000b3190e7909 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/7d0007d5193c4e69b98000b3190e7909 2024-11-19T15:15:40,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/7d0007d5193c4e69b98000b3190e7909, entries=150, sequenceid=298, filesize=12.0 K 2024-11-19T15:15:40,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 09d12e268a82f5728962d61ed7bcb9c9 in 1656ms, sequenceid=298, compaction requested=true 2024-11-19T15:15:40,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:40,431 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d12e268a82f5728962d61ed7bcb9c9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:40,431 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:40,431 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:40,431 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:40,431 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/B is initiating minor compaction (all files) 2024-11-19T15:15:40,431 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/A is initiating minor compaction (all files) 2024-11-19T15:15:40,431 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/A in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:40,431 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/B in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:40,432 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/1290a07916964ff7b4d2901894b27925, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/792cc4f1d9b2486b9ff31ef928cd3de3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/6c5e4b6bd6b74fcab811999c9698f53b] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=126.2 K 2024-11-19T15:15:40,432 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/cf52d8b4d1e44d1bba7a3a023828cc53, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/c944fa5f59b44a3d81dc43dfba546f25, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8a0485be8a0e4b3dbe366260daa0580b] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=36.7 K 2024-11-19T15:15:40,432 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:40,432 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/1290a07916964ff7b4d2901894b27925, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/792cc4f1d9b2486b9ff31ef928cd3de3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/6c5e4b6bd6b74fcab811999c9698f53b] 2024-11-19T15:15:40,432 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting cf52d8b4d1e44d1bba7a3a023828cc53, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732029335804 2024-11-19T15:15:40,432 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1290a07916964ff7b4d2901894b27925, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732029335804 2024-11-19T15:15:40,432 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting c944fa5f59b44a3d81dc43dfba546f25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732029336471 2024-11-19T15:15:40,432 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 792cc4f1d9b2486b9ff31ef928cd3de3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732029336471 2024-11-19T15:15:40,432 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c5e4b6bd6b74fcab811999c9698f53b, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732029338741 2024-11-19T15:15:40,432 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0485be8a0e4b3dbe366260daa0580b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732029338760 2024-11-19T15:15:40,438 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:40,439 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#B#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:40,439 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119e90a703b4fd540f0b8ad3192452b3006_09d12e268a82f5728962d61ed7bcb9c9 store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:40,439 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/1ee9558572f740869e32f76cab2d212a is 50, key is test_row_0/B:col10/1732029338773/Put/seqid=0 2024-11-19T15:15:40,442 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119e90a703b4fd540f0b8ad3192452b3006_09d12e268a82f5728962d61ed7bcb9c9, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:40,442 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e90a703b4fd540f0b8ad3192452b3006_09d12e268a82f5728962d61ed7bcb9c9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:40,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742259_1435 (size=13051) 2024-11-19T15:15:40,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742260_1436 (size=4469) 2024-11-19T15:15:40,846 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#A#compaction#366 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:40,847 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/18520ffc809246539594fa271dd6d0a4 is 175, key is test_row_0/A:col10/1732029338773/Put/seqid=0 2024-11-19T15:15:40,848 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/1ee9558572f740869e32f76cab2d212a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/1ee9558572f740869e32f76cab2d212a 2024-11-19T15:15:40,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742261_1437 (size=32005) 2024-11-19T15:15:40,852 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/B of 09d12e268a82f5728962d61ed7bcb9c9 into 1ee9558572f740869e32f76cab2d212a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:40,852 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:40,852 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/B, priority=13, startTime=1732029340431; duration=0sec 2024-11-19T15:15:40,853 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:40,853 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:B 2024-11-19T15:15:40,853 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:40,853 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:40,853 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 09d12e268a82f5728962d61ed7bcb9c9/C is initiating minor compaction (all files) 2024-11-19T15:15:40,854 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d12e268a82f5728962d61ed7bcb9c9/C in TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:40,854 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/da82736052c645f0ac998033ee8eb83b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/e8d1001325344a6aaaf76d431628fb5d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/7d0007d5193c4e69b98000b3190e7909] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp, totalSize=36.7 K 2024-11-19T15:15:40,854 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting da82736052c645f0ac998033ee8eb83b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732029335804 2024-11-19T15:15:40,854 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting e8d1001325344a6aaaf76d431628fb5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732029336471 2024-11-19T15:15:40,854 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d0007d5193c4e69b98000b3190e7909, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732029338760 2024-11-19T15:15:40,860 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d12e268a82f5728962d61ed7bcb9c9#C#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:40,861 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/b1591c9fa12e4db78bf35af83f52e366 is 50, key is test_row_0/C:col10/1732029338773/Put/seqid=0 2024-11-19T15:15:40,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742262_1438 (size=13051) 2024-11-19T15:15:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:41,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-19T15:15:41,036 INFO [Thread-1584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1584. 2024-11-19T15:15:41,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:41,036 DEBUG [Thread-1584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:41,036 INFO [Thread-1580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1580. 2024-11-19T15:15:41,036 INFO [Thread-1582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1582. 2024-11-19T15:15:41,036 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:41,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:41,037 DEBUG [Thread-1582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:41,037 DEBUG [Thread-1580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:41,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:41,037 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:41,037 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:41,037 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:41,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:41,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:41,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:41,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:41,037 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:41,037 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:41,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:41,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:41,038 INFO [Thread-1578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1578. 2024-11-19T15:15:41,038 DEBUG [Thread-1578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:41,038 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:41,038 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:41,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:41,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119988102eb041c48ae9795a79540f7079e_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029341034/Put/seqid=0 2024-11-19T15:15:41,045 INFO [Thread-1576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1576. 2024-11-19T15:15:41,045 DEBUG [Thread-1576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:15:41,045 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:41,045 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:41,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 158 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1059 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3177 rows 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1060 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3177 rows 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1060 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3180 rows 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1066 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3194 rows 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1062 2024-11-19T15:15:41,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3186 rows 2024-11-19T15:15:41,046 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T15:15:41,046 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T15:15:41,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:41,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:41,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:15:41,047 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T15:15:41,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742263_1439 (size=12454) 2024-11-19T15:15:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T15:15:41,050 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029341050"}]},"ts":"1732029341050"} 2024-11-19T15:15:41,052 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T15:15:41,052 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T15:15:41,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T15:15:41,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, UNASSIGN}] 2024-11-19T15:15:41,054 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, UNASSIGN 2024-11-19T15:15:41,055 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=09d12e268a82f5728962d61ed7bcb9c9, regionState=CLOSING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:41,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, UNASSIGN because future has completed 2024-11-19T15:15:41,057 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T15:15:41,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:15:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T15:15:41,209 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:41,209 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T15:15:41,209 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 09d12e268a82f5728962d61ed7bcb9c9, disabling compactions & flushes 2024-11-19T15:15:41,209 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1993): waiting for 2 compactions & cache flush to complete for region TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:41,238 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/ab3ee74bb17b4bf6982d66d5b518ca45, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a4287c5cf0442cca79415adb36c31ac, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/e853432ebc7a4f49be23b752927aa45d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/d58f98254ca6408e8d0e6679b09dc1dd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/4d8ceaa7fc9c4958803ff9d5578ec2dc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/da601fe01c69405688706de278a06f58, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/fa5c28abb95a457ba93272cf69bc57b6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/43da71a3abb84e7a8ef9c333f0c28d3c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a8d23ee391343c68d4e46f4424a233d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/c99d6e9142034f938a198a5e0e7f066a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/8c32a47700174b7896c71103e060c34a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/384bd57693f141e6907ef9079a916230, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/52a0d77d00294df89d2ffb24a97fd150, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/296ef868ab3f46b79321036468882265, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/203cc04ae08046b2b2f5500a6246d9b2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/babd566ebf87478db9a6fe84e6ad5dd8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/fc98e14b8f734687871f41d030745575, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/46cd2cb5c1184ddcb3e70d8a9ede167d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/b2e3dcdfec9e458aa145ecb9e8dede43] to archive 2024-11-19T15:15:41,239 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:15:41,240 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/ab3ee74bb17b4bf6982d66d5b518ca45 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/ab3ee74bb17b4bf6982d66d5b518ca45 2024-11-19T15:15:41,241 WARN [regionserver/50570b5b74e7:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-19T15:15:41,241 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a4287c5cf0442cca79415adb36c31ac to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a4287c5cf0442cca79415adb36c31ac 2024-11-19T15:15:41,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/e853432ebc7a4f49be23b752927aa45d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/e853432ebc7a4f49be23b752927aa45d 2024-11-19T15:15:41,243 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/d58f98254ca6408e8d0e6679b09dc1dd to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/d58f98254ca6408e8d0e6679b09dc1dd 2024-11-19T15:15:41,244 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/4d8ceaa7fc9c4958803ff9d5578ec2dc to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/4d8ceaa7fc9c4958803ff9d5578ec2dc 2024-11-19T15:15:41,245 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/da601fe01c69405688706de278a06f58 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/da601fe01c69405688706de278a06f58 2024-11-19T15:15:41,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/fa5c28abb95a457ba93272cf69bc57b6 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/fa5c28abb95a457ba93272cf69bc57b6 2024-11-19T15:15:41,247 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/43da71a3abb84e7a8ef9c333f0c28d3c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/43da71a3abb84e7a8ef9c333f0c28d3c 2024-11-19T15:15:41,247 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a8d23ee391343c68d4e46f4424a233d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/0a8d23ee391343c68d4e46f4424a233d 2024-11-19T15:15:41,248 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/c99d6e9142034f938a198a5e0e7f066a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/c99d6e9142034f938a198a5e0e7f066a 2024-11-19T15:15:41,249 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/8c32a47700174b7896c71103e060c34a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/8c32a47700174b7896c71103e060c34a 2024-11-19T15:15:41,250 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/384bd57693f141e6907ef9079a916230 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/384bd57693f141e6907ef9079a916230 2024-11-19T15:15:41,251 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/52a0d77d00294df89d2ffb24a97fd150 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/52a0d77d00294df89d2ffb24a97fd150 2024-11-19T15:15:41,252 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/296ef868ab3f46b79321036468882265 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/296ef868ab3f46b79321036468882265 2024-11-19T15:15:41,253 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/203cc04ae08046b2b2f5500a6246d9b2 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/203cc04ae08046b2b2f5500a6246d9b2 2024-11-19T15:15:41,254 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/babd566ebf87478db9a6fe84e6ad5dd8 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/babd566ebf87478db9a6fe84e6ad5dd8 2024-11-19T15:15:41,255 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/fc98e14b8f734687871f41d030745575 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/fc98e14b8f734687871f41d030745575 2024-11-19T15:15:41,255 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/18520ffc809246539594fa271dd6d0a4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/18520ffc809246539594fa271dd6d0a4 2024-11-19T15:15:41,256 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/46cd2cb5c1184ddcb3e70d8a9ede167d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/46cd2cb5c1184ddcb3e70d8a9ede167d 2024-11-19T15:15:41,257 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/b2e3dcdfec9e458aa145ecb9e8dede43 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/b2e3dcdfec9e458aa145ecb9e8dede43 2024-11-19T15:15:41,259 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/A of 09d12e268a82f5728962d61ed7bcb9c9 into 18520ffc809246539594fa271dd6d0a4(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:41,259 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:41,259 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/A, priority=13, startTime=1732029340431; duration=0sec 2024-11-19T15:15:41,259 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:41,259 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:A 2024-11-19T15:15:41,259 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/98e34250be094c8ca81f16a3b94a4018, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5cc8fb2b54924ef5afe49e89bb2dd6c9, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/e13569a61b20492bb5c3d20b8d9b146e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/dcb79ed1dbf54266896bc3dd42af2b05, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8e9773580b4f4133b17b6b56dc15f3d5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8ff5610dfd134fe3af40b1b8fbaba86d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/7f048a1c0e9540e893ab788eef63a2d6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/3b50f19490fb435b8b46753664ce30ac, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/bf3dc74f6f1245b090c6f65a608232c4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/e921038098f3427d9301bcc042da2b8e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/83300ccef3c8478f8dc8c174e4f5f2fc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/21c02e17167d40f28e1f08b442d421a3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/355845a723114fe0a7588f7674c62bbf, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8c9e2403b8fe41eb89b5d1dadca19313, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5b278f3271384a2d8d790203b87153de, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/d14cdaec8ad743329df621952834e19e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5b1f31a651564940913464ef826a8078, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/847f429c199040869700fd7e32f45760, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/cf52d8b4d1e44d1bba7a3a023828cc53, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/3f75b3b404704df8a2957bac076f1201, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/c944fa5f59b44a3d81dc43dfba546f25, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8a0485be8a0e4b3dbe366260daa0580b] to archive 2024-11-19T15:15:41,260 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:15:41,261 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/98e34250be094c8ca81f16a3b94a4018 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/98e34250be094c8ca81f16a3b94a4018 2024-11-19T15:15:41,262 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5cc8fb2b54924ef5afe49e89bb2dd6c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5cc8fb2b54924ef5afe49e89bb2dd6c9 2024-11-19T15:15:41,263 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/e13569a61b20492bb5c3d20b8d9b146e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/e13569a61b20492bb5c3d20b8d9b146e 2024-11-19T15:15:41,264 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/dcb79ed1dbf54266896bc3dd42af2b05 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/dcb79ed1dbf54266896bc3dd42af2b05 2024-11-19T15:15:41,265 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8e9773580b4f4133b17b6b56dc15f3d5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8e9773580b4f4133b17b6b56dc15f3d5 2024-11-19T15:15:41,266 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8ff5610dfd134fe3af40b1b8fbaba86d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8ff5610dfd134fe3af40b1b8fbaba86d 2024-11-19T15:15:41,267 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/7f048a1c0e9540e893ab788eef63a2d6 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/7f048a1c0e9540e893ab788eef63a2d6 2024-11-19T15:15:41,267 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/b1591c9fa12e4db78bf35af83f52e366 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/b1591c9fa12e4db78bf35af83f52e366 2024-11-19T15:15:41,267 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/3b50f19490fb435b8b46753664ce30ac to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/3b50f19490fb435b8b46753664ce30ac 2024-11-19T15:15:41,268 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/bf3dc74f6f1245b090c6f65a608232c4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/bf3dc74f6f1245b090c6f65a608232c4 2024-11-19T15:15:41,269 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/e921038098f3427d9301bcc042da2b8e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/e921038098f3427d9301bcc042da2b8e 2024-11-19T15:15:41,270 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/83300ccef3c8478f8dc8c174e4f5f2fc to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/83300ccef3c8478f8dc8c174e4f5f2fc 2024-11-19T15:15:41,271 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d12e268a82f5728962d61ed7bcb9c9/C of 09d12e268a82f5728962d61ed7bcb9c9 into b1591c9fa12e4db78bf35af83f52e366(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:41,271 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:41,271 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., storeName=09d12e268a82f5728962d61ed7bcb9c9/C, priority=13, startTime=1732029340431; duration=0sec 2024-11-19T15:15:41,271 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:41,271 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:41,271 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d12e268a82f5728962d61ed7bcb9c9:C 2024-11-19T15:15:41,271 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/21c02e17167d40f28e1f08b442d421a3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/21c02e17167d40f28e1f08b442d421a3 2024-11-19T15:15:41,272 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/355845a723114fe0a7588f7674c62bbf to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/355845a723114fe0a7588f7674c62bbf 2024-11-19T15:15:41,273 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8c9e2403b8fe41eb89b5d1dadca19313 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8c9e2403b8fe41eb89b5d1dadca19313 2024-11-19T15:15:41,274 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5b278f3271384a2d8d790203b87153de to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5b278f3271384a2d8d790203b87153de 2024-11-19T15:15:41,275 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/d14cdaec8ad743329df621952834e19e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/d14cdaec8ad743329df621952834e19e 2024-11-19T15:15:41,275 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5b1f31a651564940913464ef826a8078 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/5b1f31a651564940913464ef826a8078 2024-11-19T15:15:41,276 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/847f429c199040869700fd7e32f45760 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/847f429c199040869700fd7e32f45760 2024-11-19T15:15:41,277 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/cf52d8b4d1e44d1bba7a3a023828cc53 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/cf52d8b4d1e44d1bba7a3a023828cc53 2024-11-19T15:15:41,278 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/3f75b3b404704df8a2957bac076f1201 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/3f75b3b404704df8a2957bac076f1201 2024-11-19T15:15:41,279 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/c944fa5f59b44a3d81dc43dfba546f25 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/c944fa5f59b44a3d81dc43dfba546f25 2024-11-19T15:15:41,279 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8a0485be8a0e4b3dbe366260daa0580b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/8a0485be8a0e4b3dbe366260daa0580b 2024-11-19T15:15:41,282 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/218e0302463c456f937a9f6e5bc7e1c1, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/9b30d9cdc70d446488656b46790cf6f9, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/d872a486746b444e832b14358bc87d90, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/e3d64752990a4809803f594775af0314, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/48bab867087f454686c54c37f3c2182b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/9b0e95ba3b1f4a3a87e1e14dd0d0faab, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/f3074f247a6c4a66b13844efd961ca66, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/1c0a117d36344af4bdf11d56f8eba034, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8d356b17699c45c5bc634d83a2c6264e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/07ef6cf2f1ec4f8ba2f41216df5ee3d2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8da942bfc6954b9e88932ec352c68785, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/625186935a5242e2b219c41d78883510, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/0332d52d4b2041c3a0f955ca1a251c1d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/ba7317997a1849159b68730690a87aa2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/607d5575c4d84aa6b3566b304160719d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/747d100d6de64ecfb80f152b239e7d1d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/752770aa230d495c9a99ab268ef4119a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/0b9a8b66357a498ba4af78413e75bd43, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/da82736052c645f0ac998033ee8eb83b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8f34fda3328e48fdb4f5f8aad94acce3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/e8d1001325344a6aaaf76d431628fb5d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/7d0007d5193c4e69b98000b3190e7909] to archive 2024-11-19T15:15:41,283 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:15:41,284 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/218e0302463c456f937a9f6e5bc7e1c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/218e0302463c456f937a9f6e5bc7e1c1 2024-11-19T15:15:41,285 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/9b30d9cdc70d446488656b46790cf6f9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/9b30d9cdc70d446488656b46790cf6f9 2024-11-19T15:15:41,286 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/d872a486746b444e832b14358bc87d90 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/d872a486746b444e832b14358bc87d90 2024-11-19T15:15:41,287 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/e3d64752990a4809803f594775af0314 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/e3d64752990a4809803f594775af0314 2024-11-19T15:15:41,288 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/48bab867087f454686c54c37f3c2182b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/48bab867087f454686c54c37f3c2182b 2024-11-19T15:15:41,289 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/9b0e95ba3b1f4a3a87e1e14dd0d0faab to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/9b0e95ba3b1f4a3a87e1e14dd0d0faab 2024-11-19T15:15:41,290 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/f3074f247a6c4a66b13844efd961ca66 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/f3074f247a6c4a66b13844efd961ca66 2024-11-19T15:15:41,290 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/1c0a117d36344af4bdf11d56f8eba034 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/1c0a117d36344af4bdf11d56f8eba034 2024-11-19T15:15:41,291 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8d356b17699c45c5bc634d83a2c6264e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8d356b17699c45c5bc634d83a2c6264e 2024-11-19T15:15:41,292 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/07ef6cf2f1ec4f8ba2f41216df5ee3d2 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/07ef6cf2f1ec4f8ba2f41216df5ee3d2 2024-11-19T15:15:41,293 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8da942bfc6954b9e88932ec352c68785 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8da942bfc6954b9e88932ec352c68785 2024-11-19T15:15:41,294 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/625186935a5242e2b219c41d78883510 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/625186935a5242e2b219c41d78883510 2024-11-19T15:15:41,295 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/0332d52d4b2041c3a0f955ca1a251c1d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/0332d52d4b2041c3a0f955ca1a251c1d 2024-11-19T15:15:41,295 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/ba7317997a1849159b68730690a87aa2 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/ba7317997a1849159b68730690a87aa2 2024-11-19T15:15:41,296 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/607d5575c4d84aa6b3566b304160719d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/607d5575c4d84aa6b3566b304160719d 2024-11-19T15:15:41,297 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/747d100d6de64ecfb80f152b239e7d1d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/747d100d6de64ecfb80f152b239e7d1d 2024-11-19T15:15:41,298 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/752770aa230d495c9a99ab268ef4119a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/752770aa230d495c9a99ab268ef4119a 2024-11-19T15:15:41,299 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/0b9a8b66357a498ba4af78413e75bd43 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/0b9a8b66357a498ba4af78413e75bd43 2024-11-19T15:15:41,300 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/da82736052c645f0ac998033ee8eb83b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/da82736052c645f0ac998033ee8eb83b 2024-11-19T15:15:41,301 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8f34fda3328e48fdb4f5f8aad94acce3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/8f34fda3328e48fdb4f5f8aad94acce3 2024-11-19T15:15:41,301 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/e8d1001325344a6aaaf76d431628fb5d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/e8d1001325344a6aaaf76d431628fb5d 2024-11-19T15:15:41,302 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/50570b5b74e7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/7d0007d5193c4e69b98000b3190e7909 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/7d0007d5193c4e69b98000b3190e7909 2024-11-19T15:15:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T15:15:41,448 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:41,451 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119988102eb041c48ae9795a79540f7079e_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119988102eb041c48ae9795a79540f7079e_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:41,451 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/cb8f6520505745a4a68e036865824ed8, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:41,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/cb8f6520505745a4a68e036865824ed8 is 175, key is test_row_0/A:col10/1732029341034/Put/seqid=0 2024-11-19T15:15:41,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742264_1440 (size=31255) 2024-11-19T15:15:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T15:15:41,855 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/cb8f6520505745a4a68e036865824ed8 2024-11-19T15:15:41,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/a53c4798680149cba24dbcba568f9049 is 50, key is test_row_0/B:col10/1732029341034/Put/seqid=0 2024-11-19T15:15:41,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742265_1441 (size=12301) 2024-11-19T15:15:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T15:15:42,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/a53c4798680149cba24dbcba568f9049 2024-11-19T15:15:42,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/fe10ab6de7df49f7b2f9852dc3245420 is 50, key is test_row_0/C:col10/1732029341034/Put/seqid=0 2024-11-19T15:15:42,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742266_1442 (size=12301) 2024-11-19T15:15:42,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/fe10ab6de7df49f7b2f9852dc3245420 2024-11-19T15:15:42,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/cb8f6520505745a4a68e036865824ed8 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/cb8f6520505745a4a68e036865824ed8 2024-11-19T15:15:42,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/cb8f6520505745a4a68e036865824ed8, entries=150, sequenceid=336, filesize=30.5 K 2024-11-19T15:15:42,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/a53c4798680149cba24dbcba568f9049 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/a53c4798680149cba24dbcba568f9049 2024-11-19T15:15:42,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/a53c4798680149cba24dbcba568f9049, entries=150, sequenceid=336, filesize=12.0 K 2024-11-19T15:15:42,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/fe10ab6de7df49f7b2f9852dc3245420 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/fe10ab6de7df49f7b2f9852dc3245420 2024-11-19T15:15:42,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/fe10ab6de7df49f7b2f9852dc3245420, entries=150, sequenceid=336, filesize=12.0 K 2024-11-19T15:15:42,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.88 KB/456576, currentSize=13.42 KB/13740 for 09d12e268a82f5728962d61ed7bcb9c9 in 1652ms, sequenceid=336, compaction requested=false 2024-11-19T15:15:42,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d12e268a82f5728962d61ed7bcb9c9: 2024-11-19T15:15:42,688 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:42,688 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:42,688 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. after waiting 0 ms 2024-11-19T15:15:42,688 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:42,688 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 09d12e268a82f5728962d61ed7bcb9c9 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-19T15:15:42,689 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=A 2024-11-19T15:15:42,689 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:42,689 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=B 2024-11-19T15:15:42,689 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:42,689 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d12e268a82f5728962d61ed7bcb9c9, store=C 2024-11-19T15:15:42,689 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:42,693 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197fcab98f3a0a44b9957f7d798765d414_09d12e268a82f5728962d61ed7bcb9c9 is 50, key is test_row_0/A:col10/1732029341044/Put/seqid=0 2024-11-19T15:15:42,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742267_1443 (size=9914) 2024-11-19T15:15:43,096 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:43,099 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197fcab98f3a0a44b9957f7d798765d414_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197fcab98f3a0a44b9957f7d798765d414_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:43,100 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/193263af6c134596a36a8c8cbc5c5236, store: [table=TestAcidGuarantees family=A region=09d12e268a82f5728962d61ed7bcb9c9] 2024-11-19T15:15:43,101 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/193263af6c134596a36a8c8cbc5c5236 is 175, key is test_row_0/A:col10/1732029341044/Put/seqid=0 2024-11-19T15:15:43,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742268_1444 (size=22561) 2024-11-19T15:15:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T15:15:43,504 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/193263af6c134596a36a8c8cbc5c5236 2024-11-19T15:15:43,510 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/7df6bc94b0434d6d8aaff20efb5d67ca is 50, key is test_row_0/B:col10/1732029341044/Put/seqid=0 2024-11-19T15:15:43,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742269_1445 (size=9857) 2024-11-19T15:15:43,914 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/7df6bc94b0434d6d8aaff20efb5d67ca 2024-11-19T15:15:43,919 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/22a932a7d73f4a8e919d587a4c34d9ca is 50, key is test_row_0/C:col10/1732029341044/Put/seqid=0 2024-11-19T15:15:43,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742270_1446 (size=9857) 2024-11-19T15:15:44,323 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/22a932a7d73f4a8e919d587a4c34d9ca 2024-11-19T15:15:44,326 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/A/193263af6c134596a36a8c8cbc5c5236 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/193263af6c134596a36a8c8cbc5c5236 2024-11-19T15:15:44,329 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/193263af6c134596a36a8c8cbc5c5236, entries=100, sequenceid=343, filesize=22.0 K 2024-11-19T15:15:44,330 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/B/7df6bc94b0434d6d8aaff20efb5d67ca as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/7df6bc94b0434d6d8aaff20efb5d67ca 2024-11-19T15:15:44,333 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/7df6bc94b0434d6d8aaff20efb5d67ca, entries=100, sequenceid=343, filesize=9.6 K 2024-11-19T15:15:44,333 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/.tmp/C/22a932a7d73f4a8e919d587a4c34d9ca as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/22a932a7d73f4a8e919d587a4c34d9ca 2024-11-19T15:15:44,336 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/22a932a7d73f4a8e919d587a4c34d9ca, entries=100, sequenceid=343, filesize=9.6 K 2024-11-19T15:15:44,337 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 09d12e268a82f5728962d61ed7bcb9c9 in 1649ms, sequenceid=343, compaction requested=true 2024-11-19T15:15:44,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/1290a07916964ff7b4d2901894b27925, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/792cc4f1d9b2486b9ff31ef928cd3de3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/6c5e4b6bd6b74fcab811999c9698f53b] to archive 2024-11-19T15:15:44,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:15:44,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/1290a07916964ff7b4d2901894b27925 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/1290a07916964ff7b4d2901894b27925 2024-11-19T15:15:44,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/792cc4f1d9b2486b9ff31ef928cd3de3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/792cc4f1d9b2486b9ff31ef928cd3de3 2024-11-19T15:15:44,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/6c5e4b6bd6b74fcab811999c9698f53b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/6c5e4b6bd6b74fcab811999c9698f53b 2024-11-19T15:15:44,345 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/recovered.edits/346.seqid, newMaxSeqId=346, maxSeqId=4 2024-11-19T15:15:44,346 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. 2024-11-19T15:15:44,346 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 09d12e268a82f5728962d61ed7bcb9c9: Waiting for close lock at 1732029341209Running coprocessor pre-close hooks at 1732029341209Disabling compacts and flushes for region at 1732029341209Disabling writes for close at 1732029342688 (+1479 ms)Obtaining lock to block concurrent updates at 1732029342688Preparing flush snapshotting stores in 09d12e268a82f5728962d61ed7bcb9c9 at 1732029342688Finished memstore snapshotting TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732029342689 (+1 ms)Flushing stores of TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9. at 1732029342689Flushing 09d12e268a82f5728962d61ed7bcb9c9/A: creating writer at 1732029342689Flushing mob file 09d12e268a82f5728962d61ed7bcb9c9/A: appending metadata at 1732029342693 (+4 ms)Flushing mob file 09d12e268a82f5728962d61ed7bcb9c9/A: closing flushed file at 1732029342693Flushing 09d12e268a82f5728962d61ed7bcb9c9/A: appending metadata at 1732029343100 (+407 ms)Flushing 09d12e268a82f5728962d61ed7bcb9c9/A: closing flushed file at 1732029343100Flushing 09d12e268a82f5728962d61ed7bcb9c9/B: creating writer at 1732029343507 (+407 ms)Flushing 09d12e268a82f5728962d61ed7bcb9c9/B: appending metadata at 1732029343510 (+3 ms)Flushing 09d12e268a82f5728962d61ed7bcb9c9/B: closing flushed file at 1732029343510Flushing 09d12e268a82f5728962d61ed7bcb9c9/C: creating writer at 1732029343917 (+407 ms)Flushing 09d12e268a82f5728962d61ed7bcb9c9/C: appending metadata at 1732029343919 (+2 ms)Flushing 09d12e268a82f5728962d61ed7bcb9c9/C: closing flushed file at 1732029343919Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@560cc723: reopening flushed file at 1732029344326 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e0f87ab: reopening flushed file at 1732029344329 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d047e3f: reopening flushed file at 1732029344333 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 09d12e268a82f5728962d61ed7bcb9c9 in 1649ms, sequenceid=343, compaction requested=true at 1732029344337 (+4 ms)Writing region close event to WAL at 1732029344343 (+6 ms)Running coprocessor post-close hooks at 1732029344346 (+3 ms)Closed at 1732029344346 2024-11-19T15:15:44,348 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:44,348 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=09d12e268a82f5728962d61ed7bcb9c9, regionState=CLOSED 2024-11-19T15:15:44,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:15:44,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-19T15:15:44,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 09d12e268a82f5728962d61ed7bcb9c9, server=50570b5b74e7,46557,1732029218639 in 3.2930 sec 2024-11-19T15:15:44,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-19T15:15:44,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d12e268a82f5728962d61ed7bcb9c9, UNASSIGN in 3.2980 sec 2024-11-19T15:15:44,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-19T15:15:44,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.3010 sec 2024-11-19T15:15:44,356 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029344356"}]},"ts":"1732029344356"} 2024-11-19T15:15:44,357 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T15:15:44,357 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T15:15:44,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.3110 sec 2024-11-19T15:15:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T15:15:45,204 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:45,205 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T15:15:45,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:45,206 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T15:15:45,207 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:45,209 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,211 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/recovered.edits] 2024-11-19T15:15:45,213 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/18520ffc809246539594fa271dd6d0a4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/18520ffc809246539594fa271dd6d0a4 2024-11-19T15:15:45,214 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/193263af6c134596a36a8c8cbc5c5236 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/193263af6c134596a36a8c8cbc5c5236 2024-11-19T15:15:45,214 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/cb8f6520505745a4a68e036865824ed8 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/A/cb8f6520505745a4a68e036865824ed8 2024-11-19T15:15:45,216 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/1ee9558572f740869e32f76cab2d212a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/1ee9558572f740869e32f76cab2d212a 2024-11-19T15:15:45,217 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/7df6bc94b0434d6d8aaff20efb5d67ca to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/7df6bc94b0434d6d8aaff20efb5d67ca 2024-11-19T15:15:45,218 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/a53c4798680149cba24dbcba568f9049 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/B/a53c4798680149cba24dbcba568f9049 2024-11-19T15:15:45,220 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/22a932a7d73f4a8e919d587a4c34d9ca to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/22a932a7d73f4a8e919d587a4c34d9ca 2024-11-19T15:15:45,221 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/b1591c9fa12e4db78bf35af83f52e366 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/b1591c9fa12e4db78bf35af83f52e366 2024-11-19T15:15:45,222 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/fe10ab6de7df49f7b2f9852dc3245420 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/C/fe10ab6de7df49f7b2f9852dc3245420 2024-11-19T15:15:45,224 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/recovered.edits/346.seqid to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9/recovered.edits/346.seqid 2024-11-19T15:15:45,224 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,224 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T15:15:45,225 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T15:15:45,225 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T15:15:45,227 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190b719770134d4bd9928624129f2a1d27_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190b719770134d4bd9928624129f2a1d27_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,228 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191cbf668d37b04a4bb7080cdb4602437f_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191cbf668d37b04a4bb7080cdb4602437f_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,229 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111941e6663af9124a5abcd1cfcf50890ea2_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111941e6663af9124a5abcd1cfcf50890ea2_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,230 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194b01a52700a3413889a01380b34376c3_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194b01a52700a3413889a01380b34376c3_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,231 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195814fc2a9edd48268cef1c62d8dd6ada_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195814fc2a9edd48268cef1c62d8dd6ada_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,232 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195886914a292d4495add01bd95f74ef2d_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195886914a292d4495add01bd95f74ef2d_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,233 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197fcab98f3a0a44b9957f7d798765d414_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197fcab98f3a0a44b9957f7d798765d414_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,234 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198c98e8f704d24eb8ac7a7ede8a7c4cfd_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198c98e8f704d24eb8ac7a7ede8a7c4cfd_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,235 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119988102eb041c48ae9795a79540f7079e_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119988102eb041c48ae9795a79540f7079e_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,235 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199dbfbddb1e2840218da09f6eccd6d8fd_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199dbfbddb1e2840218da09f6eccd6d8fd_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,236 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199ed84e1974784146b13b7780a9b357fb_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199ed84e1974784146b13b7780a9b357fb_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,237 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199f4a822684fb40c38967e90441ef713d_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199f4a822684fb40c38967e90441ef713d_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,238 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a11b15fa78734d27a1e58d203ed5b6ac_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a11b15fa78734d27a1e58d203ed5b6ac_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,239 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac43eb59d9624282969798c2e6c02494_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac43eb59d9624282969798c2e6c02494_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,240 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c1fa82859ff04e7bbaf17dfaa98b77a5_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c1fa82859ff04e7bbaf17dfaa98b77a5_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,241 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c6b9a73ff9d34fbb9202159ad3b8288c_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c6b9a73ff9d34fbb9202159ad3b8288c_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,241 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ec3720cc461a4c508836e37d3648e928_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ec3720cc461a4c508836e37d3648e928_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,242 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f60f1aa99a5645c19c3da49b836d66b4_09d12e268a82f5728962d61ed7bcb9c9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f60f1aa99a5645c19c3da49b836d66b4_09d12e268a82f5728962d61ed7bcb9c9 2024-11-19T15:15:45,243 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T15:15:45,244 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:45,246 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T15:15:45,248 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T15:15:45,249 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:45,249 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T15:15:45,249 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732029345249"}]},"ts":"9223372036854775807"} 2024-11-19T15:15:45,251 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T15:15:45,251 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 09d12e268a82f5728962d61ed7bcb9c9, NAME => 'TestAcidGuarantees,,1732029316973.09d12e268a82f5728962d61ed7bcb9c9.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T15:15:45,251 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T15:15:45,251 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732029345251"}]},"ts":"9223372036854775807"} 2024-11-19T15:15:45,253 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T15:15:45,253 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:45,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-19T15:15:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T15:15:45,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T15:15:45,314 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:45,323 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 245) - Thread LEAK? -, OpenFileDescriptor=455 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=418 (was 414) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7381 (was 7873) 2024-11-19T15:15:45,332 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=418, ProcessCount=11, AvailableMemoryMB=7380 2024-11-19T15:15:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T15:15:45,334 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T15:15:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T15:15:45,335 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T15:15:45,336 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:45,336 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-19T15:15:45,336 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T15:15:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-19T15:15:45,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742271_1447 (size=1061) 2024-11-19T15:15:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-19T15:15:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-19T15:15:45,743 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8 2024-11-19T15:15:45,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742272_1448 (size=53) 2024-11-19T15:15:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-19T15:15:46,148 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:15:46,148 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9d89d22f9c07cea236aa6c26f89b6b4f, disabling compactions & flushes 2024-11-19T15:15:46,148 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:46,148 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:46,148 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. after waiting 0 ms 2024-11-19T15:15:46,148 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:46,148 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:46,148 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9d89d22f9c07cea236aa6c26f89b6b4f: Waiting for close lock at 1732029346148Disabling compacts and flushes for region at 1732029346148Disabling writes for close at 1732029346148Writing region close event to WAL at 1732029346148Closed at 1732029346148 2024-11-19T15:15:46,149 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T15:15:46,149 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732029346149"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732029346149"}]},"ts":"1732029346149"} 2024-11-19T15:15:46,151 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T15:15:46,151 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T15:15:46,152 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029346151"}]},"ts":"1732029346151"} 2024-11-19T15:15:46,153 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T15:15:46,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d89d22f9c07cea236aa6c26f89b6b4f, ASSIGN}] 2024-11-19T15:15:46,154 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d89d22f9c07cea236aa6c26f89b6b4f, ASSIGN 2024-11-19T15:15:46,154 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d89d22f9c07cea236aa6c26f89b6b4f, ASSIGN; state=OFFLINE, location=50570b5b74e7,46557,1732029218639; forceNewPlan=false, retain=false 2024-11-19T15:15:46,305 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=9d89d22f9c07cea236aa6c26f89b6b4f, regionState=OPENING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:46,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d89d22f9c07cea236aa6c26f89b6b4f, ASSIGN because future has completed 2024-11-19T15:15:46,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:15:46,461 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:46,461 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} 2024-11-19T15:15:46,462 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,462 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:15:46,462 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,462 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,463 INFO [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,464 INFO [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:15:46,464 INFO [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d89d22f9c07cea236aa6c26f89b6b4f columnFamilyName A 2024-11-19T15:15:46,465 DEBUG [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:46,465 INFO [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] regionserver.HStore(327): Store=9d89d22f9c07cea236aa6c26f89b6b4f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:15:46,465 INFO [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,466 INFO [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:15:46,466 INFO [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d89d22f9c07cea236aa6c26f89b6b4f columnFamilyName B 2024-11-19T15:15:46,466 DEBUG [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:46,466 INFO [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] regionserver.HStore(327): Store=9d89d22f9c07cea236aa6c26f89b6b4f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:15:46,466 INFO [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,467 INFO [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:15:46,467 INFO [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d89d22f9c07cea236aa6c26f89b6b4f columnFamilyName C 2024-11-19T15:15:46,467 DEBUG [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:15:46,467 INFO [StoreOpener-9d89d22f9c07cea236aa6c26f89b6b4f-1 {}] regionserver.HStore(327): Store=9d89d22f9c07cea236aa6c26f89b6b4f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:15:46,468 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:46,468 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,468 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,468 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,469 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,469 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,470 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T15:15:46,470 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,472 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T15:15:46,472 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 9d89d22f9c07cea236aa6c26f89b6b4f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59549559, jitterRate=-0.11264242231845856}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T15:15:46,472 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:46,473 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 9d89d22f9c07cea236aa6c26f89b6b4f: Running coprocessor pre-open hook at 1732029346462Writing region info on filesystem at 1732029346462Initializing all the Stores at 1732029346463 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029346463Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029346463Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029346463Cleaning up temporary data from old regions at 1732029346469 (+6 ms)Running coprocessor post-open hooks at 1732029346472 (+3 ms)Region opened successfully at 1732029346473 (+1 ms) 2024-11-19T15:15:46,473 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., pid=122, masterSystemTime=1732029346458 2024-11-19T15:15:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-19T15:15:46,475 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:46,475 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:46,475 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=9d89d22f9c07cea236aa6c26f89b6b4f, regionState=OPEN, openSeqNum=2, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:46,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:15:46,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-19T15:15:46,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 in 170 msec 2024-11-19T15:15:46,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-19T15:15:46,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d89d22f9c07cea236aa6c26f89b6b4f, ASSIGN in 325 msec 2024-11-19T15:15:46,480 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T15:15:46,480 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029346480"}]},"ts":"1732029346480"} 2024-11-19T15:15:46,481 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T15:15:46,482 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T15:15:46,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-19T15:15:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-19T15:15:47,484 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:47,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6516064e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:47,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:47,486 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f20b8ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:47,488 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b74d299, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,489 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:47,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:47,491 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T15:15:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 50570b5b74e7,39501,1732029217890 2024-11-19T15:15:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1e62f9b9 2024-11-19T15:15:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T15:15:47,492 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T15:15:47,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,494 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:47,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:47,494 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31204482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:47,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:47,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:47,495 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@151140b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:47,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:47,497 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f40cd5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:47,498 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38aa7f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:47,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:47,499 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547e8a38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:47,501 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52cb606f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:47,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:47,502 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f374177, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:47,503 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62cc27bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,504 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:47,504 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:47,505 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221a3f9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:47,506 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:47,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d321d16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:47,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:47,508 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd8cf29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:47,509 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:47,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd7a6cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:47,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:47,511 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1910016c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:47,512 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f749d49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:47,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:47,514 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:47,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:47,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:47,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75603099, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:47,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:47,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:47,515 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:47,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10cd5d12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:47,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:47,517 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d9bdd80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:47,518 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:47,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e86783b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c1b90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,519 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:15:47,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:15:47,520 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:15:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:15:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:15:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d05ce92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:15:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:15:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:15:47,521 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:15:47,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d38825f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:15:47,522 DEBUG [Thread-2021 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:47,522 DEBUG [Thread-2013 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:47,522 DEBUG [Thread-2015 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:47,523 DEBUG [Thread-2030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:47,523 DEBUG [Thread-2028 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:47,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:47,523 DEBUG [Thread-2019 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:47,523 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:47,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:47,524 DEBUG [Thread-2024 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:47,524 DEBUG [Thread-2017 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:47,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:47,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:47,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:47,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:47,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:47,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:47,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:47,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:47,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:47,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:47,526 DEBUG [Thread-2024 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:47,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:47,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:47,526 DEBUG [Thread-2024 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:47,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:47,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:47,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:47,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:47,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:47,527 DEBUG [Thread-2026 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:47,527 DEBUG [Thread-2032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:15:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:15:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:15:47,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:15:47,530 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:15:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:15:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:15:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:15:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:15:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:15:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:15:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:15:47,533 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:15:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-19T15:15:47,534 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2] 2024-11-19T15:15:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T15:15:47,536 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:47,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:47,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:15:47,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:47,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:47,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:47,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:47,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:47,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:47,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/1e70096196b34112b8a0f301bea06958 is 50, key is test_row_0/A:col10/1732029347538/Put/seqid=0 2024-11-19T15:15:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742273_1449 (size=14337) 2024-11-19T15:15:47,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029347572, max timestamp=1732029347600Num uniques 859; 2024-11-19T15:15:47,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:15:47,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029407573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029347572, max timestamp=1732029347600Num uniques 859; 2024-11-19T15:15:47,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:15:47,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029347572, max timestamp=1732029347600Num uniques 859; 2024-11-19T15:15:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:15:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T15:15:47,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029407639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029407641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029407647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029407648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T15:15:47,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:47,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:47,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:47,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:47,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:47,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029407714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029407754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029407754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029407765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029407765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T15:15:47,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:47,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:47,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:47,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:47,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T15:15:47,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029407924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029407964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029407964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/1e70096196b34112b8a0f301bea06958 2024-11-19T15:15:47,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029407974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:47,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029407974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:47,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:47,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:47,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/71d0ab68b8754b6b8dcb3b75e1f449bb is 50, key is test_row_0/B:col10/1732029347538/Put/seqid=0 2024-11-19T15:15:47,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T15:15:47,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:47,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:47,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:47,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:47,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:47,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742274_1450 (size=9657) 2024-11-19T15:15:47,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/71d0ab68b8754b6b8dcb3b75e1f449bb 2024-11-19T15:15:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:48,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/d94c0ea785b849438e862e1c8ebcbeec is 50, key is test_row_0/C:col10/1732029347538/Put/seqid=0 2024-11-19T15:15:48,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742275_1451 (size=9657) 2024-11-19T15:15:48,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/d94c0ea785b849438e862e1c8ebcbeec 2024-11-19T15:15:48,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/1e70096196b34112b8a0f301bea06958 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/1e70096196b34112b8a0f301bea06958 2024-11-19T15:15:48,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/1e70096196b34112b8a0f301bea06958, entries=200, sequenceid=12, filesize=14.0 K 2024-11-19T15:15:48,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/71d0ab68b8754b6b8dcb3b75e1f449bb as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/71d0ab68b8754b6b8dcb3b75e1f449bb 2024-11-19T15:15:48,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/71d0ab68b8754b6b8dcb3b75e1f449bb, entries=100, sequenceid=12, filesize=9.4 K 2024-11-19T15:15:48,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/d94c0ea785b849438e862e1c8ebcbeec as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/d94c0ea785b849438e862e1c8ebcbeec 2024-11-19T15:15:48,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/d94c0ea785b849438e862e1c8ebcbeec, entries=100, sequenceid=12, filesize=9.4 K 2024-11-19T15:15:48,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9d89d22f9c07cea236aa6c26f89b6b4f in 504ms, sequenceid=12, compaction requested=false 2024-11-19T15:15:48,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:48,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T15:15:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:48,148 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-19T15:15:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:48,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/cabc2c11c79d434c810ce5cf1e4c0eed is 50, key is test_row_0/A:col10/1732029347646/Put/seqid=0 2024-11-19T15:15:48,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742276_1452 (size=12001) 2024-11-19T15:15:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T15:15:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:48,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:48,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029408277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029408286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029408287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029408293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029408294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029408394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029408395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029408404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,564 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/cabc2c11c79d434c810ce5cf1e4c0eed 2024-11-19T15:15:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/3e66d60aa7a64af5aa96c43ccdded87b is 50, key is test_row_0/B:col10/1732029347646/Put/seqid=0 2024-11-19T15:15:48,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742277_1453 (size=12001) 2024-11-19T15:15:48,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029408604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029408604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029408624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T15:15:48,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029408814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029408815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029408914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029408915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029408944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:48,976 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/3e66d60aa7a64af5aa96c43ccdded87b 2024-11-19T15:15:48,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/8e68fb017a334999bb8ab2fbfcb340fe is 50, key is test_row_0/C:col10/1732029347646/Put/seqid=0 2024-11-19T15:15:48,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742278_1454 (size=12001) 2024-11-19T15:15:49,386 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/8e68fb017a334999bb8ab2fbfcb340fe 2024-11-19T15:15:49,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/cabc2c11c79d434c810ce5cf1e4c0eed as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/cabc2c11c79d434c810ce5cf1e4c0eed 2024-11-19T15:15:49,393 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/cabc2c11c79d434c810ce5cf1e4c0eed, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T15:15:49,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/3e66d60aa7a64af5aa96c43ccdded87b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/3e66d60aa7a64af5aa96c43ccdded87b 2024-11-19T15:15:49,397 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/3e66d60aa7a64af5aa96c43ccdded87b, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T15:15:49,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/8e68fb017a334999bb8ab2fbfcb340fe as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/8e68fb017a334999bb8ab2fbfcb340fe 2024-11-19T15:15:49,401 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/8e68fb017a334999bb8ab2fbfcb340fe, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T15:15:49,402 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 9d89d22f9c07cea236aa6c26f89b6b4f in 1253ms, sequenceid=48, compaction requested=false 2024-11-19T15:15:49,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:49,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:49,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-19T15:15:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-19T15:15:49,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-19T15:15:49,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8670 sec 2024-11-19T15:15:49,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.8830 sec 2024-11-19T15:15:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:49,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T15:15:49,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:49,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:49,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:49,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:49,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:49,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:49,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/4f41d71caece48fbbf711b9f75d536ef is 50, key is test_row_0/A:col10/1732029348285/Put/seqid=0 2024-11-19T15:15:49,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742279_1455 (size=14341) 2024-11-19T15:15:49,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029409491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:49,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029409492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029409501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:49,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029409614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:49,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029409614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029409614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T15:15:49,684 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:49,685 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-19T15:15:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T15:15:49,686 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:49,687 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:49,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T15:15:49,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T15:15:49,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:49,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:49,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:49,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:49,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:49,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029409834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:49,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029409834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:49,845 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 9d89d22f9c07cea236aa6c26f89b6b4f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:49,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029409835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:49,845 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 9d89d22f9c07cea236aa6c26f89b6b4f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:15:49,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029409835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:49,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029409835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:49,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/4f41d71caece48fbbf711b9f75d536ef 2024-11-19T15:15:49,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:49,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/77a77781e0e54f65b49017995b34e2a7 is 50, key is test_row_0/B:col10/1732029348285/Put/seqid=0 2024-11-19T15:15:49,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742280_1456 (size=12001) 2024-11-19T15:15:49,868 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T15:15:49,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T15:15:49,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:49,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:49,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:49,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:49,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T15:15:50,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T15:15:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:50,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:50,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:50,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029410154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:50,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029410155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:50,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029410155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:50,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/77a77781e0e54f65b49017995b34e2a7 2024-11-19T15:15:50,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/9a3555b885ae4f029c50c8ab2dadc0ed is 50, key is test_row_0/C:col10/1732029348285/Put/seqid=0 2024-11-19T15:15:50,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742281_1457 (size=12001) 2024-11-19T15:15:50,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/9a3555b885ae4f029c50c8ab2dadc0ed 2024-11-19T15:15:50,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/4f41d71caece48fbbf711b9f75d536ef as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/4f41d71caece48fbbf711b9f75d536ef 2024-11-19T15:15:50,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/4f41d71caece48fbbf711b9f75d536ef, entries=200, sequenceid=60, filesize=14.0 K 2024-11-19T15:15:50,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/77a77781e0e54f65b49017995b34e2a7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/77a77781e0e54f65b49017995b34e2a7 2024-11-19T15:15:50,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/77a77781e0e54f65b49017995b34e2a7, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T15:15:50,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/9a3555b885ae4f029c50c8ab2dadc0ed as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/9a3555b885ae4f029c50c8ab2dadc0ed 2024-11-19T15:15:50,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/9a3555b885ae4f029c50c8ab2dadc0ed, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T15:15:50,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9d89d22f9c07cea236aa6c26f89b6b4f in 854ms, sequenceid=60, compaction requested=true 2024-11-19T15:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:50,293 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:50,293 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:50,294 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:50,294 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40679 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:50,294 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/B is initiating minor compaction (all files) 2024-11-19T15:15:50,294 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/A is initiating minor compaction (all files) 2024-11-19T15:15:50,294 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/A in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:50,294 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/B in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:50,294 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/1e70096196b34112b8a0f301bea06958, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/cabc2c11c79d434c810ce5cf1e4c0eed, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/4f41d71caece48fbbf711b9f75d536ef] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=39.7 K 2024-11-19T15:15:50,294 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/71d0ab68b8754b6b8dcb3b75e1f449bb, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/3e66d60aa7a64af5aa96c43ccdded87b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/77a77781e0e54f65b49017995b34e2a7] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=32.9 K 2024-11-19T15:15:50,294 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e70096196b34112b8a0f301bea06958, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732029347538 2024-11-19T15:15:50,294 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 71d0ab68b8754b6b8dcb3b75e1f449bb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732029347538 2024-11-19T15:15:50,295 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e66d60aa7a64af5aa96c43ccdded87b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732029347630 2024-11-19T15:15:50,295 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting cabc2c11c79d434c810ce5cf1e4c0eed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732029347630 2024-11-19T15:15:50,295 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f41d71caece48fbbf711b9f75d536ef, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029348254 2024-11-19T15:15:50,295 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 77a77781e0e54f65b49017995b34e2a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029348271 2024-11-19T15:15:50,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T15:15:50,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:50,302 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T15:15:50,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:50,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:50,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:50,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:50,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:50,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:50,307 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#B#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:50,307 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#A#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:50,307 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/75eb0879b66b4d45a9ddc3a1d686c872 is 50, key is test_row_0/B:col10/1732029348285/Put/seqid=0 2024-11-19T15:15:50,307 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/099809012e43451e9380fbd6f533051c is 50, key is test_row_0/A:col10/1732029348285/Put/seqid=0 2024-11-19T15:15:50,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/af311936d1964281bfd998a0883eb3a0 is 50, key is test_row_0/A:col10/1732029349488/Put/seqid=0 2024-11-19T15:15:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T15:15:50,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742283_1459 (size=12001) 2024-11-19T15:15:50,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742284_1460 (size=12104) 2024-11-19T15:15:50,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742282_1458 (size=12104) 2024-11-19T15:15:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:50,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:50,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029410698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029410706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:50,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029410707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:50,718 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/af311936d1964281bfd998a0883eb3a0 2024-11-19T15:15:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/23292933bac04a45b31bc61c32ecac6d is 50, key is test_row_0/B:col10/1732029349488/Put/seqid=0 2024-11-19T15:15:50,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742285_1461 (size=12001) 2024-11-19T15:15:50,733 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/23292933bac04a45b31bc61c32ecac6d 2024-11-19T15:15:50,735 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/75eb0879b66b4d45a9ddc3a1d686c872 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/75eb0879b66b4d45a9ddc3a1d686c872 2024-11-19T15:15:50,737 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/099809012e43451e9380fbd6f533051c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/099809012e43451e9380fbd6f533051c 2024-11-19T15:15:50,741 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/B of 9d89d22f9c07cea236aa6c26f89b6b4f into 75eb0879b66b4d45a9ddc3a1d686c872(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:50,741 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:50,741 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/B, priority=13, startTime=1732029350293; duration=0sec 2024-11-19T15:15:50,741 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:50,742 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:B 2024-11-19T15:15:50,742 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:50,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/c8289d55b95d4263996774d12b9792a5 is 50, key is test_row_0/C:col10/1732029349488/Put/seqid=0 2024-11-19T15:15:50,746 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/A of 9d89d22f9c07cea236aa6c26f89b6b4f into 099809012e43451e9380fbd6f533051c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:50,746 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:50,746 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/A, priority=13, startTime=1732029350293; duration=0sec 2024-11-19T15:15:50,746 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:50,746 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:50,746 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:A 2024-11-19T15:15:50,746 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/C is initiating minor compaction (all files) 2024-11-19T15:15:50,746 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/C in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:50,746 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/d94c0ea785b849438e862e1c8ebcbeec, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/8e68fb017a334999bb8ab2fbfcb340fe, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/9a3555b885ae4f029c50c8ab2dadc0ed] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=32.9 K 2024-11-19T15:15:50,746 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d94c0ea785b849438e862e1c8ebcbeec, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732029347538 2024-11-19T15:15:50,747 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e68fb017a334999bb8ab2fbfcb340fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732029347630 2024-11-19T15:15:50,747 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a3555b885ae4f029c50c8ab2dadc0ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029348271 2024-11-19T15:15:50,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742286_1462 (size=12001) 2024-11-19T15:15:50,750 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/c8289d55b95d4263996774d12b9792a5 2024-11-19T15:15:50,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/af311936d1964281bfd998a0883eb3a0 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/af311936d1964281bfd998a0883eb3a0 2024-11-19T15:15:50,758 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#C#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:50,759 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/78ee9404fdf14810a081d277bdfd988d is 50, key is test_row_0/C:col10/1732029348285/Put/seqid=0 2024-11-19T15:15:50,760 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/af311936d1964281bfd998a0883eb3a0, entries=150, sequenceid=80, filesize=11.7 K 2024-11-19T15:15:50,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/23292933bac04a45b31bc61c32ecac6d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/23292933bac04a45b31bc61c32ecac6d 2024-11-19T15:15:50,764 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/23292933bac04a45b31bc61c32ecac6d, entries=150, sequenceid=80, filesize=11.7 K 2024-11-19T15:15:50,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742287_1463 (size=12104) 2024-11-19T15:15:50,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/c8289d55b95d4263996774d12b9792a5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c8289d55b95d4263996774d12b9792a5 2024-11-19T15:15:50,770 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c8289d55b95d4263996774d12b9792a5, entries=150, sequenceid=80, filesize=11.7 K 2024-11-19T15:15:50,771 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9d89d22f9c07cea236aa6c26f89b6b4f in 469ms, sequenceid=80, compaction requested=false 2024-11-19T15:15:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-19T15:15:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-19T15:15:50,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-19T15:15:50,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0850 sec 2024-11-19T15:15:50,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.0890 sec 2024-11-19T15:15:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:50,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T15:15:50,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:50,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:50,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:50,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:50,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:50,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T15:15:50,824 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:50,825 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:50,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/f3b73a9a21014f5ca40a44056c93befd is 50, key is test_row_0/A:col10/1732029350696/Put/seqid=0 2024-11-19T15:15:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-19T15:15:50,830 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T15:15:50,831 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:50,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:50,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742288_1464 (size=14341) 2024-11-19T15:15:50,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/f3b73a9a21014f5ca40a44056c93befd 2024-11-19T15:15:50,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/37720bc958e5490fbcb2137dd1cdd19d is 50, key is test_row_0/B:col10/1732029350696/Put/seqid=0 2024-11-19T15:15:50,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742289_1465 (size=12001) 2024-11-19T15:15:50,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/37720bc958e5490fbcb2137dd1cdd19d 2024-11-19T15:15:50,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/cf17485b961242139f181c6bcfe5283f is 50, key is test_row_0/C:col10/1732029350696/Put/seqid=0 2024-11-19T15:15:50,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742290_1466 (size=12001) 2024-11-19T15:15:50,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/cf17485b961242139f181c6bcfe5283f 2024-11-19T15:15:50,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/f3b73a9a21014f5ca40a44056c93befd as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/f3b73a9a21014f5ca40a44056c93befd 2024-11-19T15:15:50,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/f3b73a9a21014f5ca40a44056c93befd, entries=200, sequenceid=95, filesize=14.0 K 2024-11-19T15:15:50,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/37720bc958e5490fbcb2137dd1cdd19d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/37720bc958e5490fbcb2137dd1cdd19d 2024-11-19T15:15:50,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/37720bc958e5490fbcb2137dd1cdd19d, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T15:15:50,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/cf17485b961242139f181c6bcfe5283f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/cf17485b961242139f181c6bcfe5283f 2024-11-19T15:15:50,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/cf17485b961242139f181c6bcfe5283f, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T15:15:50,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=73.80 KB/75570 for 9d89d22f9c07cea236aa6c26f89b6b4f in 61ms, sequenceid=95, compaction requested=true 2024-11-19T15:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:A, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:B, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T15:15:50,881 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:50,882 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:50,882 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/A is initiating minor compaction (all files) 2024-11-19T15:15:50,882 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/A in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:50,882 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/099809012e43451e9380fbd6f533051c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/af311936d1964281bfd998a0883eb3a0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/f3b73a9a21014f5ca40a44056c93befd] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=37.5 K 2024-11-19T15:15:50,883 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 099809012e43451e9380fbd6f533051c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029348271 2024-11-19T15:15:50,883 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting af311936d1964281bfd998a0883eb3a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732029349488 2024-11-19T15:15:50,883 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3b73a9a21014f5ca40a44056c93befd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732029350695 2024-11-19T15:15:50,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T15:15:50,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:50,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:50,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:50,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:50,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:50,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:50,890 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#A#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:50,890 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/73f4022f820b48f38cb5df8b9070b2fd is 50, key is test_row_0/A:col10/1732029350696/Put/seqid=0 2024-11-19T15:15:50,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/57cc6699130c4b52b0f7f29438a3ce54 is 50, key is test_row_0/A:col10/1732029350888/Put/seqid=0 2024-11-19T15:15:50,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742291_1467 (size=12207) 2024-11-19T15:15:50,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742292_1468 (size=14341) 2024-11-19T15:15:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T15:15:50,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029410936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:50,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029410938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:50,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029410945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:50,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:50,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:50,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:50,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:50,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:50,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:50,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:50,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029411054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029411054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029411064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:51,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:51,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:51,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T15:15:51,169 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/78ee9404fdf14810a081d277bdfd988d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/78ee9404fdf14810a081d277bdfd988d 2024-11-19T15:15:51,174 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/C of 9d89d22f9c07cea236aa6c26f89b6b4f into 78ee9404fdf14810a081d277bdfd988d(size=11.8 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:51,174 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:51,174 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/C, priority=13, startTime=1732029350293; duration=0sec 2024-11-19T15:15:51,174 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T15:15:51,174 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:C 2024-11-19T15:15:51,174 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:C 2024-11-19T15:15:51,174 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:51,175 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:51,175 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/B is initiating minor compaction (all files) 2024-11-19T15:15:51,175 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/B in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,175 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/75eb0879b66b4d45a9ddc3a1d686c872, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/23292933bac04a45b31bc61c32ecac6d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/37720bc958e5490fbcb2137dd1cdd19d] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=35.3 K 2024-11-19T15:15:51,176 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 75eb0879b66b4d45a9ddc3a1d686c872, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029348271 2024-11-19T15:15:51,176 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 23292933bac04a45b31bc61c32ecac6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732029349488 2024-11-19T15:15:51,176 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 37720bc958e5490fbcb2137dd1cdd19d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732029350695 2024-11-19T15:15:51,181 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:51,182 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/25e7accce89f46d5a9cff4beea6cbc1c is 50, key is test_row_0/B:col10/1732029350696/Put/seqid=0 2024-11-19T15:15:51,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742293_1469 (size=12207) 2024-11-19T15:15:51,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029411274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029411274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:51,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029411275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:51,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:51,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:51,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/57cc6699130c4b52b0f7f29438a3ce54 2024-11-19T15:15:51,306 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/73f4022f820b48f38cb5df8b9070b2fd as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/73f4022f820b48f38cb5df8b9070b2fd 2024-11-19T15:15:51,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/70c8cac92157479494d4d4870e18e2cd is 50, key is test_row_0/B:col10/1732029350888/Put/seqid=0 2024-11-19T15:15:51,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742294_1470 (size=12001) 2024-11-19T15:15:51,323 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/A of 9d89d22f9c07cea236aa6c26f89b6b4f into 73f4022f820b48f38cb5df8b9070b2fd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:51,323 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:51,323 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/A, priority=13, startTime=1732029350881; duration=0sec 2024-11-19T15:15:51,323 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:51,323 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:A 2024-11-19T15:15:51,323 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:51,324 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:51,324 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/C is initiating minor compaction (all files) 2024-11-19T15:15:51,324 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/C in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,324 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/78ee9404fdf14810a081d277bdfd988d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c8289d55b95d4263996774d12b9792a5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/cf17485b961242139f181c6bcfe5283f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=35.3 K 2024-11-19T15:15:51,324 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78ee9404fdf14810a081d277bdfd988d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732029348271 2024-11-19T15:15:51,325 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8289d55b95d4263996774d12b9792a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732029349488 2024-11-19T15:15:51,325 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf17485b961242139f181c6bcfe5283f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732029350695 2024-11-19T15:15:51,333 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#C#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:51,334 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/c54e910543284ce59d45e7c86eebcc48 is 50, key is test_row_0/C:col10/1732029350696/Put/seqid=0 2024-11-19T15:15:51,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742295_1471 (size=12207) 2024-11-19T15:15:51,344 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/c54e910543284ce59d45e7c86eebcc48 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c54e910543284ce59d45e7c86eebcc48 2024-11-19T15:15:51,349 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/C of 9d89d22f9c07cea236aa6c26f89b6b4f into c54e910543284ce59d45e7c86eebcc48(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:51,349 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:51,349 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/C, priority=13, startTime=1732029350881; duration=0sec 2024-11-19T15:15:51,349 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:51,349 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:C 2024-11-19T15:15:51,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:51,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:51,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T15:15:51,593 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/25e7accce89f46d5a9cff4beea6cbc1c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/25e7accce89f46d5a9cff4beea6cbc1c 2024-11-19T15:15:51,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:51,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:51,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,597 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/B of 9d89d22f9c07cea236aa6c26f89b6b4f into 25e7accce89f46d5a9cff4beea6cbc1c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:51,597 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:51,597 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/B, priority=13, startTime=1732029350881; duration=0sec 2024-11-19T15:15:51,597 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:51,597 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:B 2024-11-19T15:15:51,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029411594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029411594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029411594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:51,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/70c8cac92157479494d4d4870e18e2cd 2024-11-19T15:15:51,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/48d3fad64cf442fd8f106b5e7bbe6a0b is 50, key is test_row_0/C:col10/1732029350888/Put/seqid=0 2024-11-19T15:15:51,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742296_1472 (size=12001) 2024-11-19T15:15:51,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/48d3fad64cf442fd8f106b5e7bbe6a0b 2024-11-19T15:15:51,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/57cc6699130c4b52b0f7f29438a3ce54 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/57cc6699130c4b52b0f7f29438a3ce54 2024-11-19T15:15:51,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/57cc6699130c4b52b0f7f29438a3ce54, entries=200, sequenceid=110, filesize=14.0 K 2024-11-19T15:15:51,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/70c8cac92157479494d4d4870e18e2cd as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/70c8cac92157479494d4d4870e18e2cd 2024-11-19T15:15:51,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/70c8cac92157479494d4d4870e18e2cd, entries=150, sequenceid=110, filesize=11.7 K 2024-11-19T15:15:51,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/48d3fad64cf442fd8f106b5e7bbe6a0b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/48d3fad64cf442fd8f106b5e7bbe6a0b 2024-11-19T15:15:51,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:51,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/48d3fad64cf442fd8f106b5e7bbe6a0b, entries=150, sequenceid=110, filesize=11.7 K 2024-11-19T15:15:51,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:51,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9d89d22f9c07cea236aa6c26f89b6b4f in 860ms, sequenceid=110, compaction requested=false 2024-11-19T15:15:51,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:51,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T15:15:51,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:51,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:51,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:51,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:51,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:51,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:51,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/b49aa668766a47c08669571047ee4da5 is 50, key is test_row_0/A:col10/1732029350943/Put/seqid=0 2024-11-19T15:15:51,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742297_1473 (size=14391) 2024-11-19T15:15:51,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/b49aa668766a47c08669571047ee4da5 2024-11-19T15:15:51,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/0239a4d81621423784c84593abb28ed5 is 50, key is test_row_0/B:col10/1732029350943/Put/seqid=0 2024-11-19T15:15:51,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742298_1474 (size=12051) 2024-11-19T15:15:51,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:51,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:51,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:51,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:51,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029411936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:51,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029411936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T15:15:52,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029412054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:52,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029412054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:52,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029412114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:52,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029412114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:52,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029412115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:52,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:52,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:52,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029412274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:52,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029412274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:52,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/0239a4d81621423784c84593abb28ed5 2024-11-19T15:15:52,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/c0204e182ac34c7fb8baad81ac20565b is 50, key is test_row_0/C:col10/1732029350943/Put/seqid=0 2024-11-19T15:15:52,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742299_1475 (size=12051) 2024-11-19T15:15:52,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:52,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:52,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029412594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029412594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:52,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:52,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:52,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:52,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:52,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/c0204e182ac34c7fb8baad81ac20565b 2024-11-19T15:15:52,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/b49aa668766a47c08669571047ee4da5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/b49aa668766a47c08669571047ee4da5 2024-11-19T15:15:52,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/b49aa668766a47c08669571047ee4da5, entries=200, sequenceid=132, filesize=14.1 K 2024-11-19T15:15:52,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/0239a4d81621423784c84593abb28ed5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/0239a4d81621423784c84593abb28ed5 2024-11-19T15:15:52,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/0239a4d81621423784c84593abb28ed5, entries=150, sequenceid=132, filesize=11.8 K 2024-11-19T15:15:52,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/c0204e182ac34c7fb8baad81ac20565b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c0204e182ac34c7fb8baad81ac20565b 2024-11-19T15:15:52,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c0204e182ac34c7fb8baad81ac20565b, entries=150, sequenceid=132, filesize=11.8 K 2024-11-19T15:15:52,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 9d89d22f9c07cea236aa6c26f89b6b4f in 855ms, sequenceid=132, compaction requested=true 2024-11-19T15:15:52,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:52,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:52,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:52,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:52,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:52,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:52,715 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:52,716 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:52,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:52,716 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:52,716 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/B is initiating minor compaction (all files) 2024-11-19T15:15:52,717 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/B in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,717 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/25e7accce89f46d5a9cff4beea6cbc1c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/70c8cac92157479494d4d4870e18e2cd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/0239a4d81621423784c84593abb28ed5] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=35.4 K 2024-11-19T15:15:52,717 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40939 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:52,717 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/A is initiating minor compaction (all files) 2024-11-19T15:15:52,717 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/A in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,717 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/73f4022f820b48f38cb5df8b9070b2fd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/57cc6699130c4b52b0f7f29438a3ce54, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/b49aa668766a47c08669571047ee4da5] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=40.0 K 2024-11-19T15:15:52,717 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 25e7accce89f46d5a9cff4beea6cbc1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732029350695 2024-11-19T15:15:52,717 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73f4022f820b48f38cb5df8b9070b2fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732029350695 2024-11-19T15:15:52,718 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 70c8cac92157479494d4d4870e18e2cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732029350871 2024-11-19T15:15:52,718 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57cc6699130c4b52b0f7f29438a3ce54, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732029350871 2024-11-19T15:15:52,718 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 0239a4d81621423784c84593abb28ed5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732029350927 2024-11-19T15:15:52,718 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting b49aa668766a47c08669571047ee4da5, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732029350927 2024-11-19T15:15:52,724 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#A#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:52,725 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/e4f34fd2df0f4bab85f0790d1aad1383 is 50, key is test_row_0/A:col10/1732029350943/Put/seqid=0 2024-11-19T15:15:52,726 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:52,727 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/cd3763cf94e843c29a1000469ba0b182 is 50, key is test_row_0/B:col10/1732029350943/Put/seqid=0 2024-11-19T15:15:52,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742300_1476 (size=12359) 2024-11-19T15:15:52,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742301_1477 (size=12359) 2024-11-19T15:15:52,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T15:15:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:52,820 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T15:15:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:52,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/2745b0661e1945b48ab2f66879994a19 is 50, key is test_row_0/A:col10/1732029351894/Put/seqid=0 2024-11-19T15:15:52,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742302_1478 (size=12151) 2024-11-19T15:15:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T15:15:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:53,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:53,139 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/e4f34fd2df0f4bab85f0790d1aad1383 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/e4f34fd2df0f4bab85f0790d1aad1383 2024-11-19T15:15:53,139 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/cd3763cf94e843c29a1000469ba0b182 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/cd3763cf94e843c29a1000469ba0b182 2024-11-19T15:15:53,143 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/B of 9d89d22f9c07cea236aa6c26f89b6b4f into cd3763cf94e843c29a1000469ba0b182(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:53,143 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/A of 9d89d22f9c07cea236aa6c26f89b6b4f into e4f34fd2df0f4bab85f0790d1aad1383(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:53,143 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:53,143 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:53,143 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/B, priority=13, startTime=1732029352715; duration=0sec 2024-11-19T15:15:53,143 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/A, priority=13, startTime=1732029352715; duration=0sec 2024-11-19T15:15:53,143 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:53,143 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:B 2024-11-19T15:15:53,143 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:53,143 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:A 2024-11-19T15:15:53,143 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:53,144 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:53,144 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/C is initiating minor compaction (all files) 2024-11-19T15:15:53,144 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/C in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:53,144 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c54e910543284ce59d45e7c86eebcc48, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/48d3fad64cf442fd8f106b5e7bbe6a0b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c0204e182ac34c7fb8baad81ac20565b] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=35.4 K 2024-11-19T15:15:53,145 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting c54e910543284ce59d45e7c86eebcc48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732029350695 2024-11-19T15:15:53,145 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 48d3fad64cf442fd8f106b5e7bbe6a0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732029350871 2024-11-19T15:15:53,146 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting c0204e182ac34c7fb8baad81ac20565b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732029350927 2024-11-19T15:15:53,153 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:53,153 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/db2445bb7a934fd7845c6d974aaeef41 is 50, key is test_row_0/C:col10/1732029350943/Put/seqid=0 2024-11-19T15:15:53,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742303_1479 (size=12359) 2024-11-19T15:15:53,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029413149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029413162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029413164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029413165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029413165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,228 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/2745b0661e1945b48ab2f66879994a19 2024-11-19T15:15:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/0b695affab174d258fc0c038685ff57c is 50, key is test_row_0/B:col10/1732029351894/Put/seqid=0 2024-11-19T15:15:53,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742304_1480 (size=12151) 2024-11-19T15:15:53,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029413274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029413284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029413284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029413284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029413284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029413494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029413504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029413504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029413505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029413505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,560 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/db2445bb7a934fd7845c6d974aaeef41 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/db2445bb7a934fd7845c6d974aaeef41 2024-11-19T15:15:53,564 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/C of 9d89d22f9c07cea236aa6c26f89b6b4f into db2445bb7a934fd7845c6d974aaeef41(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:53,564 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:53,564 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/C, priority=13, startTime=1732029352715; duration=0sec 2024-11-19T15:15:53,564 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:53,565 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:C 2024-11-19T15:15:53,639 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/0b695affab174d258fc0c038685ff57c 2024-11-19T15:15:53,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/de9222a9c3eb4e71864e95e07d78d44d is 50, key is test_row_0/C:col10/1732029351894/Put/seqid=0 2024-11-19T15:15:53,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742305_1481 (size=12151) 2024-11-19T15:15:53,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029413804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029413814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029413814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029413824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029413825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,050 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/de9222a9c3eb4e71864e95e07d78d44d 2024-11-19T15:15:54,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/2745b0661e1945b48ab2f66879994a19 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2745b0661e1945b48ab2f66879994a19 2024-11-19T15:15:54,057 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2745b0661e1945b48ab2f66879994a19, entries=150, sequenceid=147, filesize=11.9 K 2024-11-19T15:15:54,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/0b695affab174d258fc0c038685ff57c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/0b695affab174d258fc0c038685ff57c 2024-11-19T15:15:54,061 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/0b695affab174d258fc0c038685ff57c, entries=150, sequenceid=147, filesize=11.9 K 2024-11-19T15:15:54,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/de9222a9c3eb4e71864e95e07d78d44d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/de9222a9c3eb4e71864e95e07d78d44d 2024-11-19T15:15:54,064 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/de9222a9c3eb4e71864e95e07d78d44d, entries=150, sequenceid=147, filesize=11.9 K 2024-11-19T15:15:54,065 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 9d89d22f9c07cea236aa6c26f89b6b4f in 1245ms, sequenceid=147, compaction requested=false 2024-11-19T15:15:54,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:54,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:54,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-19T15:15:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-19T15:15:54,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-19T15:15:54,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2360 sec 2024-11-19T15:15:54,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.2450 sec 2024-11-19T15:15:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:54,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T15:15:54,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:54,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:54,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:54,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:54,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:54,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:54,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/97e5b37b4bb04bb2bf4aa3cfd0c77437 is 50, key is test_row_0/A:col10/1732029354328/Put/seqid=0 2024-11-19T15:15:54,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742306_1482 (size=14541) 2024-11-19T15:15:54,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029414354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029414355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029414357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029414358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029414360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029414474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029414475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029414484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029414484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029414484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029414694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029414695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029414695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029414695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029414695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:54,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/97e5b37b4bb04bb2bf4aa3cfd0c77437 2024-11-19T15:15:54,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/cdd4402b4dd543dd9bb664fb322dfcfb is 50, key is test_row_0/B:col10/1732029354328/Put/seqid=0 2024-11-19T15:15:54,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742307_1483 (size=12151) 2024-11-19T15:15:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T15:15:54,994 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:54,995 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-19T15:15:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T15:15:54,996 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:54,997 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:54,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:55,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029415014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:55,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029415014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029415014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029415014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:55,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029415015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T15:15:55,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T15:15:55,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:55,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:55,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:55,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:55,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:55,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/cdd4402b4dd543dd9bb664fb322dfcfb 2024-11-19T15:15:55,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/d6a0b13240e74888a63e618fdd481198 is 50, key is test_row_0/C:col10/1732029354328/Put/seqid=0 2024-11-19T15:15:55,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742308_1484 (size=12151) 2024-11-19T15:15:55,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T15:15:55,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:55,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:55,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:55,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:55,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T15:15:55,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T15:15:55,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:55,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:55,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:55,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:55,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:55,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029415524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:55,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029415524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:55,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029415534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:55,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:55,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029415534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:55,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029415534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:55,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/d6a0b13240e74888a63e618fdd481198 2024-11-19T15:15:55,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/97e5b37b4bb04bb2bf4aa3cfd0c77437 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/97e5b37b4bb04bb2bf4aa3cfd0c77437 2024-11-19T15:15:55,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/97e5b37b4bb04bb2bf4aa3cfd0c77437, entries=200, sequenceid=169, filesize=14.2 K 2024-11-19T15:15:55,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/cdd4402b4dd543dd9bb664fb322dfcfb as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/cdd4402b4dd543dd9bb664fb322dfcfb 2024-11-19T15:15:55,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/cdd4402b4dd543dd9bb664fb322dfcfb, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T15:15:55,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/d6a0b13240e74888a63e618fdd481198 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/d6a0b13240e74888a63e618fdd481198 2024-11-19T15:15:55,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/d6a0b13240e74888a63e618fdd481198, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T15:15:55,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9d89d22f9c07cea236aa6c26f89b6b4f in 1251ms, sequenceid=169, compaction requested=true 2024-11-19T15:15:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:55,582 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:55,582 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:55,582 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:55,583 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:55,583 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/A is initiating minor compaction (all files) 2024-11-19T15:15:55,583 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/B is initiating minor compaction (all files) 2024-11-19T15:15:55,583 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/A in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:55,583 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/B in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:55,583 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/e4f34fd2df0f4bab85f0790d1aad1383, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2745b0661e1945b48ab2f66879994a19, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/97e5b37b4bb04bb2bf4aa3cfd0c77437] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=38.1 K 2024-11-19T15:15:55,583 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/cd3763cf94e843c29a1000469ba0b182, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/0b695affab174d258fc0c038685ff57c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/cdd4402b4dd543dd9bb664fb322dfcfb] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=35.8 K 2024-11-19T15:15:55,583 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4f34fd2df0f4bab85f0790d1aad1383, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732029350927 2024-11-19T15:15:55,583 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting cd3763cf94e843c29a1000469ba0b182, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732029350927 2024-11-19T15:15:55,583 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b695affab174d258fc0c038685ff57c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732029351894 2024-11-19T15:15:55,583 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2745b0661e1945b48ab2f66879994a19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732029351894 2024-11-19T15:15:55,584 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd4402b4dd543dd9bb664fb322dfcfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732029353130 2024-11-19T15:15:55,584 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97e5b37b4bb04bb2bf4aa3cfd0c77437, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732029353130 2024-11-19T15:15:55,592 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:55,592 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/f629d7fc53a04ca89d35e0c03a9fafae is 50, key is test_row_0/B:col10/1732029354328/Put/seqid=0 2024-11-19T15:15:55,595 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#A#compaction#410 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:55,595 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/91ecdd8aa12a459d9d6a4dca761e528c is 50, key is test_row_0/A:col10/1732029354328/Put/seqid=0 2024-11-19T15:15:55,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742309_1485 (size=12561) 2024-11-19T15:15:55,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T15:15:55,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:55,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742310_1486 (size=12561) 2024-11-19T15:15:55,609 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T15:15:55,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:55,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:55,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:55,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:55,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:55,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:55,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/66dbba890d1745a9bc63379e67326195 is 50, key is test_row_0/A:col10/1732029354359/Put/seqid=0 2024-11-19T15:15:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T15:15:55,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742311_1487 (size=12151) 2024-11-19T15:15:56,005 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/f629d7fc53a04ca89d35e0c03a9fafae as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/f629d7fc53a04ca89d35e0c03a9fafae 2024-11-19T15:15:56,009 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/B of 9d89d22f9c07cea236aa6c26f89b6b4f into f629d7fc53a04ca89d35e0c03a9fafae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:56,009 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:56,009 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/B, priority=13, startTime=1732029355582; duration=0sec 2024-11-19T15:15:56,009 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:56,009 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:B 2024-11-19T15:15:56,009 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:56,010 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:56,010 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/C is initiating minor compaction (all files) 2024-11-19T15:15:56,010 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/C in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:56,010 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/db2445bb7a934fd7845c6d974aaeef41, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/de9222a9c3eb4e71864e95e07d78d44d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/d6a0b13240e74888a63e618fdd481198] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=35.8 K 2024-11-19T15:15:56,011 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting db2445bb7a934fd7845c6d974aaeef41, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732029350927 2024-11-19T15:15:56,011 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting de9222a9c3eb4e71864e95e07d78d44d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732029351894 2024-11-19T15:15:56,011 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d6a0b13240e74888a63e618fdd481198, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732029353130 2024-11-19T15:15:56,013 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/91ecdd8aa12a459d9d6a4dca761e528c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/91ecdd8aa12a459d9d6a4dca761e528c 2024-11-19T15:15:56,019 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#C#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:56,019 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/2eb54edd21bd410d8fc37a0eda036f0e is 50, key is test_row_0/C:col10/1732029354328/Put/seqid=0 2024-11-19T15:15:56,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742312_1488 (size=12561) 2024-11-19T15:15:56,026 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/66dbba890d1745a9bc63379e67326195 2024-11-19T15:15:56,028 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/A of 9d89d22f9c07cea236aa6c26f89b6b4f into 91ecdd8aa12a459d9d6a4dca761e528c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:56,028 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:56,028 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/A, priority=13, startTime=1732029355582; duration=0sec 2024-11-19T15:15:56,028 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:56,028 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:A 2024-11-19T15:15:56,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/a1f50cb6e433485b8a82beda0ba8edb6 is 50, key is test_row_0/B:col10/1732029354359/Put/seqid=0 2024-11-19T15:15:56,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742313_1489 (size=12151) 2024-11-19T15:15:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T15:15:56,430 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/2eb54edd21bd410d8fc37a0eda036f0e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/2eb54edd21bd410d8fc37a0eda036f0e 2024-11-19T15:15:56,434 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/C of 9d89d22f9c07cea236aa6c26f89b6b4f into 2eb54edd21bd410d8fc37a0eda036f0e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:56,434 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:56,434 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/C, priority=13, startTime=1732029355582; duration=0sec 2024-11-19T15:15:56,434 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:56,434 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:C 2024-11-19T15:15:56,438 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/a1f50cb6e433485b8a82beda0ba8edb6 2024-11-19T15:15:56,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/ebfb097c07574a47a8237526fa7c5df7 is 50, key is test_row_0/C:col10/1732029354359/Put/seqid=0 2024-11-19T15:15:56,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742314_1490 (size=12151) 2024-11-19T15:15:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:56,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029416577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029416579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029416580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029416585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029416592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029416704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029416704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029416704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029416704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029416704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,850 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/ebfb097c07574a47a8237526fa7c5df7 2024-11-19T15:15:56,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/66dbba890d1745a9bc63379e67326195 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/66dbba890d1745a9bc63379e67326195 2024-11-19T15:15:56,857 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/66dbba890d1745a9bc63379e67326195, entries=150, sequenceid=182, filesize=11.9 K 2024-11-19T15:15:56,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/a1f50cb6e433485b8a82beda0ba8edb6 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/a1f50cb6e433485b8a82beda0ba8edb6 2024-11-19T15:15:56,861 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/a1f50cb6e433485b8a82beda0ba8edb6, entries=150, sequenceid=182, filesize=11.9 K 2024-11-19T15:15:56,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/ebfb097c07574a47a8237526fa7c5df7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/ebfb097c07574a47a8237526fa7c5df7 2024-11-19T15:15:56,865 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/ebfb097c07574a47a8237526fa7c5df7, entries=150, sequenceid=182, filesize=11.9 K 2024-11-19T15:15:56,866 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9d89d22f9c07cea236aa6c26f89b6b4f in 1257ms, sequenceid=182, compaction requested=false 2024-11-19T15:15:56,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:56,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:56,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-19T15:15:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-19T15:15:56,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-19T15:15:56,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8700 sec 2024-11-19T15:15:56,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.8740 sec 2024-11-19T15:15:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:56,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T15:15:56,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:56,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:56,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:56,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:56,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:56,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:56,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/ac57fbf280414f0d834cbd34f3688cca is 50, key is test_row_0/A:col10/1732029356578/Put/seqid=0 2024-11-19T15:15:56,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742315_1491 (size=14541) 2024-11-19T15:15:56,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/ac57fbf280414f0d834cbd34f3688cca 2024-11-19T15:15:56,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029416933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/df3f94736a2e422c8179578b664d7224 is 50, key is test_row_0/B:col10/1732029356578/Put/seqid=0 2024-11-19T15:15:56,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742316_1492 (size=12151) 2024-11-19T15:15:56,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029416933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029416935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029416936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029416937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029417044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029417054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029417054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029417054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029417064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T15:15:57,144 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:57,145 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-19T15:15:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T15:15:57,147 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:57,147 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:57,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T15:15:57,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029417264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029417274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029417275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029417275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029417284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T15:15:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:57,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:57,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/df3f94736a2e422c8179578b664d7224 2024-11-19T15:15:57,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/26a06ca09b6548699aca0a539838b468 is 50, key is test_row_0/C:col10/1732029356578/Put/seqid=0 2024-11-19T15:15:57,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742317_1493 (size=12151) 2024-11-19T15:15:57,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T15:15:57,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:57,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:57,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:57,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:57,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T15:15:57,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029417584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029417585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029417585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029417594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T15:15:57,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:57,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:57,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:57,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:57,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:57,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029417604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:57,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:57,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:57,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/26a06ca09b6548699aca0a539838b468 2024-11-19T15:15:57,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/ac57fbf280414f0d834cbd34f3688cca as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/ac57fbf280414f0d834cbd34f3688cca 2024-11-19T15:15:57,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/ac57fbf280414f0d834cbd34f3688cca, entries=200, sequenceid=205, filesize=14.2 K 2024-11-19T15:15:57,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/df3f94736a2e422c8179578b664d7224 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/df3f94736a2e422c8179578b664d7224 2024-11-19T15:15:57,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T15:15:57,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:57,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:57,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:57,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:57,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:57,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/df3f94736a2e422c8179578b664d7224, entries=150, sequenceid=205, filesize=11.9 K 2024-11-19T15:15:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/26a06ca09b6548699aca0a539838b468 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/26a06ca09b6548699aca0a539838b468 2024-11-19T15:15:57,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/26a06ca09b6548699aca0a539838b468, entries=150, sequenceid=205, filesize=11.9 K 2024-11-19T15:15:57,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9d89d22f9c07cea236aa6c26f89b6b4f in 846ms, sequenceid=205, compaction requested=true 2024-11-19T15:15:57,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:57,767 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:57,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:15:57,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:15:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:57,768 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:15:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:57,768 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:57,768 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:57,768 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/A is initiating minor compaction (all files) 2024-11-19T15:15:57,768 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/B is initiating minor compaction (all files) 2024-11-19T15:15:57,769 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/A in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:57,769 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/B in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:57,769 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/91ecdd8aa12a459d9d6a4dca761e528c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/66dbba890d1745a9bc63379e67326195, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/ac57fbf280414f0d834cbd34f3688cca] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=38.3 K 2024-11-19T15:15:57,769 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/f629d7fc53a04ca89d35e0c03a9fafae, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/a1f50cb6e433485b8a82beda0ba8edb6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/df3f94736a2e422c8179578b664d7224] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=36.0 K 2024-11-19T15:15:57,769 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 91ecdd8aa12a459d9d6a4dca761e528c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732029353130 2024-11-19T15:15:57,769 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting f629d7fc53a04ca89d35e0c03a9fafae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732029353130 2024-11-19T15:15:57,769 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f50cb6e433485b8a82beda0ba8edb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732029354352 2024-11-19T15:15:57,769 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 66dbba890d1745a9bc63379e67326195, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732029354352 2024-11-19T15:15:57,770 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting ac57fbf280414f0d834cbd34f3688cca, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732029356577 2024-11-19T15:15:57,770 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting df3f94736a2e422c8179578b664d7224, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732029356578 2024-11-19T15:15:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T15:15:57,775 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#B#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:57,775 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/8f7f456f45774f19a5cd8657c01215e2 is 50, key is test_row_0/B:col10/1732029356578/Put/seqid=0 2024-11-19T15:15:57,778 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#A#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:57,779 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/89c17910005940faab5a3b19c63cf816 is 50, key is test_row_0/A:col10/1732029356578/Put/seqid=0 2024-11-19T15:15:57,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742318_1494 (size=12663) 2024-11-19T15:15:57,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742319_1495 (size=12663) 2024-11-19T15:15:57,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T15:15:57,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:57,913 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T15:15:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:15:57,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/2414150219d04f3baecce42c1d17878e is 50, key is test_row_0/A:col10/1732029356933/Put/seqid=0 2024-11-19T15:15:57,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742320_1496 (size=12151) 2024-11-19T15:15:58,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:58,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:58,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029418141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029418151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029418152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029418153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029418154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,186 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/8f7f456f45774f19a5cd8657c01215e2 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/8f7f456f45774f19a5cd8657c01215e2 2024-11-19T15:15:58,186 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/89c17910005940faab5a3b19c63cf816 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/89c17910005940faab5a3b19c63cf816 2024-11-19T15:15:58,190 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/A of 9d89d22f9c07cea236aa6c26f89b6b4f into 89c17910005940faab5a3b19c63cf816(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:58,190 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/B of 9d89d22f9c07cea236aa6c26f89b6b4f into 8f7f456f45774f19a5cd8657c01215e2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:58,190 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:58,190 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:58,190 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/A, priority=13, startTime=1732029357766; duration=0sec 2024-11-19T15:15:58,190 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/B, priority=13, startTime=1732029357767; duration=0sec 2024-11-19T15:15:58,190 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:58,190 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:15:58,190 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:A 2024-11-19T15:15:58,190 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:B 2024-11-19T15:15:58,190 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:15:58,191 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:15:58,191 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/C is initiating minor compaction (all files) 2024-11-19T15:15:58,191 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/C in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:58,191 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/2eb54edd21bd410d8fc37a0eda036f0e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/ebfb097c07574a47a8237526fa7c5df7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/26a06ca09b6548699aca0a539838b468] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=36.0 K 2024-11-19T15:15:58,191 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eb54edd21bd410d8fc37a0eda036f0e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732029353130 2024-11-19T15:15:58,192 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting ebfb097c07574a47a8237526fa7c5df7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732029354352 2024-11-19T15:15:58,192 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 26a06ca09b6548699aca0a539838b468, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732029356578 2024-11-19T15:15:58,197 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:15:58,198 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/e7d33660b3a840619721341d31d2f53e is 50, key is test_row_0/C:col10/1732029356578/Put/seqid=0 2024-11-19T15:15:58,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742321_1497 (size=12663) 2024-11-19T15:15:58,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029418264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029418274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029418274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029418274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029418275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T15:15:58,321 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/2414150219d04f3baecce42c1d17878e 2024-11-19T15:15:58,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/6430493983294626b558d57c2a2d9793 is 50, key is test_row_0/B:col10/1732029356933/Put/seqid=0 2024-11-19T15:15:58,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742322_1498 (size=12151) 2024-11-19T15:15:58,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029418484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029418484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029418485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029418485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029418494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,605 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/e7d33660b3a840619721341d31d2f53e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/e7d33660b3a840619721341d31d2f53e 2024-11-19T15:15:58,608 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/C of 9d89d22f9c07cea236aa6c26f89b6b4f into e7d33660b3a840619721341d31d2f53e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:15:58,609 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:58,609 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/C, priority=13, startTime=1732029357768; duration=0sec 2024-11-19T15:15:58,609 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:15:58,609 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:C 2024-11-19T15:15:58,732 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/6430493983294626b558d57c2a2d9793 2024-11-19T15:15:58,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/2296b12dbfc04e3bb52b91999c7d928c is 50, key is test_row_0/C:col10/1732029356933/Put/seqid=0 2024-11-19T15:15:58,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742323_1499 (size=12151) 2024-11-19T15:15:58,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029418804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029418804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029418804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029418805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:58,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029418814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,151 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/2296b12dbfc04e3bb52b91999c7d928c 2024-11-19T15:15:59,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/2414150219d04f3baecce42c1d17878e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2414150219d04f3baecce42c1d17878e 2024-11-19T15:15:59,159 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2414150219d04f3baecce42c1d17878e, entries=150, sequenceid=217, filesize=11.9 K 2024-11-19T15:15:59,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/6430493983294626b558d57c2a2d9793 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/6430493983294626b558d57c2a2d9793 2024-11-19T15:15:59,163 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/6430493983294626b558d57c2a2d9793, entries=150, sequenceid=217, filesize=11.9 K 2024-11-19T15:15:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/2296b12dbfc04e3bb52b91999c7d928c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/2296b12dbfc04e3bb52b91999c7d928c 2024-11-19T15:15:59,168 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/2296b12dbfc04e3bb52b91999c7d928c, entries=150, sequenceid=217, filesize=11.9 K 2024-11-19T15:15:59,169 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9d89d22f9c07cea236aa6c26f89b6b4f in 1257ms, sequenceid=217, compaction requested=false 2024-11-19T15:15:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:15:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-19T15:15:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-19T15:15:59,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-19T15:15:59,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0240 sec 2024-11-19T15:15:59,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.0280 sec 2024-11-19T15:15:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T15:15:59,295 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:15:59,295 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:15:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-19T15:15:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T15:15:59,297 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:15:59,297 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:15:59,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:15:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:15:59,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T15:15:59,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:15:59,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:59,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:15:59,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:59,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:15:59,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:15:59,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/c65e30b44abc4ec38c605ac874bc0522 is 50, key is test_row_0/A:col10/1732029358153/Put/seqid=0 2024-11-19T15:15:59,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742324_1500 (size=14541) 2024-11-19T15:15:59,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029419333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029419334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029419334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029419346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029419346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T15:15:59,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T15:15:59,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:59,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:59,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:59,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:59,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:59,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029419455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029419455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029419455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029419455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T15:15:59,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:59,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:59,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:59,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:59,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:59,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T15:15:59,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029419674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029419674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029419674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:15:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029419674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:15:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:15:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:15:59,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/c65e30b44abc4ec38c605ac874bc0522 2024-11-19T15:15:59,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/69a1227e37c24c42ad66894146c76206 is 50, key is test_row_0/B:col10/1732029358153/Put/seqid=0 2024-11-19T15:15:59,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742325_1501 (size=12151) 2024-11-19T15:15:59,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T15:15:59,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:59,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:59,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:59,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:59,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:59,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T15:15:59,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:59,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:15:59,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:15:59,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:59,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:15:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T15:16:00,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029419994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029419994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029419995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029419995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T15:16:00,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:00,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:00,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:00,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/69a1227e37c24c42ad66894146c76206 2024-11-19T15:16:00,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/e551e7e849034c62b8958d0789163066 is 50, key is test_row_0/C:col10/1732029358153/Put/seqid=0 2024-11-19T15:16:00,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742326_1502 (size=12151) 2024-11-19T15:16:00,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/e551e7e849034c62b8958d0789163066 2024-11-19T15:16:00,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/c65e30b44abc4ec38c605ac874bc0522 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/c65e30b44abc4ec38c605ac874bc0522 2024-11-19T15:16:00,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/c65e30b44abc4ec38c605ac874bc0522, entries=200, sequenceid=241, filesize=14.2 K 2024-11-19T15:16:00,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/69a1227e37c24c42ad66894146c76206 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/69a1227e37c24c42ad66894146c76206 2024-11-19T15:16:00,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/69a1227e37c24c42ad66894146c76206, entries=150, sequenceid=241, filesize=11.9 K 2024-11-19T15:16:00,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/e551e7e849034c62b8958d0789163066 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/e551e7e849034c62b8958d0789163066 2024-11-19T15:16:00,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/e551e7e849034c62b8958d0789163066, entries=150, sequenceid=241, filesize=11.9 K 2024-11-19T15:16:00,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 9d89d22f9c07cea236aa6c26f89b6b4f in 886ms, sequenceid=241, compaction requested=true 2024-11-19T15:16:00,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:00,203 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:16:00,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:16:00,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:00,203 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:16:00,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:16:00,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:00,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:16:00,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:00,204 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:16:00,204 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/A is initiating minor compaction (all files) 2024-11-19T15:16:00,204 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/A in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,204 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/89c17910005940faab5a3b19c63cf816, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2414150219d04f3baecce42c1d17878e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/c65e30b44abc4ec38c605ac874bc0522] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=38.4 K 2024-11-19T15:16:00,205 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89c17910005940faab5a3b19c63cf816, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732029356578 2024-11-19T15:16:00,205 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2414150219d04f3baecce42c1d17878e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732029356931 2024-11-19T15:16:00,205 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting c65e30b44abc4ec38c605ac874bc0522, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732029358134 2024-11-19T15:16:00,206 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:16:00,206 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/B is initiating minor compaction (all files) 2024-11-19T15:16:00,206 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/B in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,206 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/8f7f456f45774f19a5cd8657c01215e2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/6430493983294626b558d57c2a2d9793, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/69a1227e37c24c42ad66894146c76206] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=36.1 K 2024-11-19T15:16:00,207 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f7f456f45774f19a5cd8657c01215e2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732029356578 2024-11-19T15:16:00,207 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 6430493983294626b558d57c2a2d9793, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732029356931 2024-11-19T15:16:00,213 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 69a1227e37c24c42ad66894146c76206, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732029358152 2024-11-19T15:16:00,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T15:16:00,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,216 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T15:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:00,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/5879bb5db5f040fa89a2f2a219743c46 is 50, key is test_row_0/A:col10/1732029359344/Put/seqid=0 2024-11-19T15:16:00,243 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:00,244 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/7c7dc42b9bf34df8a0756e0dccd1f317 is 50, key is test_row_0/B:col10/1732029358153/Put/seqid=0 2024-11-19T15:16:00,245 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#A#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:00,246 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/9c3ccef1bd2543c29aaffc5208a008b8 is 50, key is test_row_0/A:col10/1732029358153/Put/seqid=0 2024-11-19T15:16:00,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742327_1503 (size=12151) 2024-11-19T15:16:00,252 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/5879bb5db5f040fa89a2f2a219743c46 2024-11-19T15:16:00,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742328_1504 (size=12765) 2024-11-19T15:16:00,279 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/9c3ccef1bd2543c29aaffc5208a008b8 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/9c3ccef1bd2543c29aaffc5208a008b8 2024-11-19T15:16:00,283 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/A of 9d89d22f9c07cea236aa6c26f89b6b4f into 9c3ccef1bd2543c29aaffc5208a008b8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:00,283 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:00,283 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/A, priority=13, startTime=1732029360203; duration=0sec 2024-11-19T15:16:00,283 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:00,283 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:A 2024-11-19T15:16:00,284 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:16:00,284 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:16:00,284 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/C is initiating minor compaction (all files) 2024-11-19T15:16:00,285 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/C in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,285 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/e7d33660b3a840619721341d31d2f53e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/2296b12dbfc04e3bb52b91999c7d928c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/e551e7e849034c62b8958d0789163066] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=36.1 K 2024-11-19T15:16:00,286 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7d33660b3a840619721341d31d2f53e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732029356578 2024-11-19T15:16:00,288 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2296b12dbfc04e3bb52b91999c7d928c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732029356931 2024-11-19T15:16:00,288 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e551e7e849034c62b8958d0789163066, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732029358152 2024-11-19T15:16:00,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/7ab59a6af08d4309b95c27ceba8ad23f is 50, key is test_row_0/B:col10/1732029359344/Put/seqid=0 2024-11-19T15:16:00,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742329_1505 (size=12765) 2024-11-19T15:16:00,312 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/7c7dc42b9bf34df8a0756e0dccd1f317 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7c7dc42b9bf34df8a0756e0dccd1f317 2024-11-19T15:16:00,317 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/B of 9d89d22f9c07cea236aa6c26f89b6b4f into 7c7dc42b9bf34df8a0756e0dccd1f317(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:00,317 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:00,317 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/B, priority=13, startTime=1732029360203; duration=0sec 2024-11-19T15:16:00,317 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:00,317 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:B 2024-11-19T15:16:00,323 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#C#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:00,324 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/a713a652aa61461a86b09e996b2def01 is 50, key is test_row_0/C:col10/1732029358153/Put/seqid=0 2024-11-19T15:16:00,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742330_1506 (size=12151) 2024-11-19T15:16:00,342 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/7ab59a6af08d4309b95c27ceba8ad23f 2024-11-19T15:16:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/6bc134eeb73348edac4a1748d746d0ee is 50, key is test_row_0/C:col10/1732029359344/Put/seqid=0 2024-11-19T15:16:00,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742331_1507 (size=12765) 2024-11-19T15:16:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:16:00,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:00,373 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/a713a652aa61461a86b09e996b2def01 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/a713a652aa61461a86b09e996b2def01 2024-11-19T15:16:00,380 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/C of 9d89d22f9c07cea236aa6c26f89b6b4f into a713a652aa61461a86b09e996b2def01(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:00,380 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:00,380 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/C, priority=13, startTime=1732029360203; duration=0sec 2024-11-19T15:16:00,380 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:00,380 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:C 2024-11-19T15:16:00,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742332_1508 (size=12151) 2024-11-19T15:16:00,391 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/6bc134eeb73348edac4a1748d746d0ee 2024-11-19T15:16:00,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/5879bb5db5f040fa89a2f2a219743c46 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/5879bb5db5f040fa89a2f2a219743c46 2024-11-19T15:16:00,400 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/5879bb5db5f040fa89a2f2a219743c46, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T15:16:00,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/7ab59a6af08d4309b95c27ceba8ad23f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7ab59a6af08d4309b95c27ceba8ad23f 2024-11-19T15:16:00,409 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7ab59a6af08d4309b95c27ceba8ad23f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T15:16:00,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/6bc134eeb73348edac4a1748d746d0ee as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/6bc134eeb73348edac4a1748d746d0ee 2024-11-19T15:16:00,416 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/6bc134eeb73348edac4a1748d746d0ee, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T15:16:00,417 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=40.25 KB/41220 for 9d89d22f9c07cea236aa6c26f89b6b4f in 201ms, sequenceid=253, compaction requested=false 2024-11-19T15:16:00,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:00,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-19T15:16:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-19T15:16:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:16:00,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:16:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:16:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:16:00,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:00,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:16:00,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:00,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-19T15:16:00,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1210 sec 2024-11-19T15:16:00,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.1280 sec 2024-11-19T15:16:00,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/22ba5b8f28e84026a708cb85628de02f is 50, key is test_row_0/A:col10/1732029360408/Put/seqid=0 2024-11-19T15:16:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T15:16:00,434 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:00,435 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:16:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-19T15:16:00,440 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:16:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T15:16:00,441 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:16:00,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:16:00,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742333_1509 (size=12301) 2024-11-19T15:16:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T15:16:00,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029420531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029360521, max timestamp=1732029360544Num uniques 859; 2024-11-19T15:16:00,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:16:00,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029360521, max timestamp=1732029360544Num uniques 859; 2024-11-19T15:16:00,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:16:00,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029360521, max timestamp=1732029360544Num uniques 859; 2024-11-19T15:16:00,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:16:00,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:00,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029420584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029420588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029420599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029420600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029420654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029420704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029420714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029420724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029420724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:00,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:00,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:00,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T15:16:00,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/22ba5b8f28e84026a708cb85628de02f 2024-11-19T15:16:00,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/2c2f467d65b340319426ce7e1caa9e44 is 50, key is test_row_0/B:col10/1732029360408/Put/seqid=0 2024-11-19T15:16:00,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742334_1510 (size=12301) 2024-11-19T15:16:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029420873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:00,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:00,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:00,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:00,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:00,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029420924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029420934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029420944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029420945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:00,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:01,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:01,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T15:16:01,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029421184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:01,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:01,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029421244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029421254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:01,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/2c2f467d65b340319426ce7e1caa9e44 2024-11-19T15:16:01,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029421264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:01,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029421264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/1308eed68faf4a7e90ab22943a779769 is 50, key is test_row_0/C:col10/1732029360408/Put/seqid=0 2024-11-19T15:16:01,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742335_1511 (size=12301) 2024-11-19T15:16:01,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:01,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:01,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T15:16:01,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:01,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:01,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/1308eed68faf4a7e90ab22943a779769 2024-11-19T15:16:01,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/22ba5b8f28e84026a708cb85628de02f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/22ba5b8f28e84026a708cb85628de02f 2024-11-19T15:16:01,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/22ba5b8f28e84026a708cb85628de02f, entries=150, sequenceid=266, filesize=12.0 K 2024-11-19T15:16:01,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/2c2f467d65b340319426ce7e1caa9e44 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/2c2f467d65b340319426ce7e1caa9e44 2024-11-19T15:16:01,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/2c2f467d65b340319426ce7e1caa9e44, entries=150, sequenceid=266, filesize=12.0 K 2024-11-19T15:16:01,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/1308eed68faf4a7e90ab22943a779769 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/1308eed68faf4a7e90ab22943a779769 2024-11-19T15:16:01,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/1308eed68faf4a7e90ab22943a779769, entries=150, sequenceid=266, filesize=12.0 K 2024-11-19T15:16:01,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9d89d22f9c07cea236aa6c26f89b6b4f in 1274ms, sequenceid=266, compaction requested=true 2024-11-19T15:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:01,695 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:16:01,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:01,695 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:16:01,696 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:16:01,696 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/A is initiating minor compaction (all files) 2024-11-19T15:16:01,696 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/A in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,696 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/9c3ccef1bd2543c29aaffc5208a008b8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/5879bb5db5f040fa89a2f2a219743c46, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/22ba5b8f28e84026a708cb85628de02f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=36.3 K 2024-11-19T15:16:01,696 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:16:01,696 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c3ccef1bd2543c29aaffc5208a008b8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732029358152 2024-11-19T15:16:01,696 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/B is initiating minor compaction (all files) 2024-11-19T15:16:01,696 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/B in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,696 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7c7dc42b9bf34df8a0756e0dccd1f317, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7ab59a6af08d4309b95c27ceba8ad23f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/2c2f467d65b340319426ce7e1caa9e44] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=36.3 K 2024-11-19T15:16:01,696 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5879bb5db5f040fa89a2f2a219743c46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732029359329 2024-11-19T15:16:01,697 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c7dc42b9bf34df8a0756e0dccd1f317, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732029358152 2024-11-19T15:16:01,697 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22ba5b8f28e84026a708cb85628de02f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732029360403 2024-11-19T15:16:01,697 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ab59a6af08d4309b95c27ceba8ad23f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732029359329 2024-11-19T15:16:01,697 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c2f467d65b340319426ce7e1caa9e44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732029360403 2024-11-19T15:16:01,704 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#A#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:01,704 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#B#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:01,705 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/89f68965dac44ec9b1ee723863bf5b91 is 50, key is test_row_0/A:col10/1732029360408/Put/seqid=0 2024-11-19T15:16:01,705 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/496c54d041314f56a776ec0958dad3cc is 50, key is test_row_0/B:col10/1732029360408/Put/seqid=0 2024-11-19T15:16:01,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742336_1512 (size=13017) 2024-11-19T15:16:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:16:01,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T15:16:01,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:16:01,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:01,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:16:01,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:01,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:16:01,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:01,721 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/89f68965dac44ec9b1ee723863bf5b91 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/89f68965dac44ec9b1ee723863bf5b91 2024-11-19T15:16:01,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/7daf1f095b394083ab46688fb26a4d90 is 50, key is test_row_0/A:col10/1732029361713/Put/seqid=0 2024-11-19T15:16:01,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742337_1513 (size=13017) 2024-11-19T15:16:01,727 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/A of 9d89d22f9c07cea236aa6c26f89b6b4f into 89f68965dac44ec9b1ee723863bf5b91(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:01,727 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:01,727 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/A, priority=13, startTime=1732029361695; duration=0sec 2024-11-19T15:16:01,727 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:01,727 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:A 2024-11-19T15:16:01,727 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:16:01,729 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/496c54d041314f56a776ec0958dad3cc as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/496c54d041314f56a776ec0958dad3cc 2024-11-19T15:16:01,729 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:16:01,729 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/C is initiating minor compaction (all files) 2024-11-19T15:16:01,729 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/C in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,729 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/a713a652aa61461a86b09e996b2def01, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/6bc134eeb73348edac4a1748d746d0ee, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/1308eed68faf4a7e90ab22943a779769] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=36.3 K 2024-11-19T15:16:01,730 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting a713a652aa61461a86b09e996b2def01, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732029358152 2024-11-19T15:16:01,731 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bc134eeb73348edac4a1748d746d0ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732029359329 2024-11-19T15:16:01,752 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1308eed68faf4a7e90ab22943a779769, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732029360403 2024-11-19T15:16:01,769 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/B of 9d89d22f9c07cea236aa6c26f89b6b4f into 496c54d041314f56a776ec0958dad3cc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:01,769 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:01,769 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/B, priority=13, startTime=1732029361695; duration=0sec 2024-11-19T15:16:01,769 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:01,769 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:B 2024-11-19T15:16:01,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742338_1514 (size=14741) 2024-11-19T15:16:01,775 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#C#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:01,776 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/78ecee4865b14ce49bc58eb461162e20 is 50, key is test_row_0/C:col10/1732029360408/Put/seqid=0 2024-11-19T15:16:01,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029421774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:01,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029421780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029421784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:01,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029421784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742339_1515 (size=13017) 2024-11-19T15:16:01,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029421792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:01,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:01,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029421894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029421914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:01,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:01,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:01,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:01,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029422104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:02,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:02,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:02,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:02,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:02,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029422134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:02,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/7daf1f095b394083ab46688fb26a4d90 2024-11-19T15:16:02,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/7db462762c484f038f37273a38dfdc78 is 50, key is test_row_0/B:col10/1732029361713/Put/seqid=0 2024-11-19T15:16:02,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742340_1516 (size=12301) 2024-11-19T15:16:02,209 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/78ecee4865b14ce49bc58eb461162e20 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/78ecee4865b14ce49bc58eb461162e20 2024-11-19T15:16:02,213 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/C of 9d89d22f9c07cea236aa6c26f89b6b4f into 78ecee4865b14ce49bc58eb461162e20(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:02,213 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:02,213 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/C, priority=13, startTime=1732029361695; duration=0sec 2024-11-19T15:16:02,213 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:02,213 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:C 2024-11-19T15:16:02,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:02,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:02,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:02,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:02,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:02,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:02,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:02,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:02,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029422424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:02,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:02,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:02,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029422454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T15:16:02,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/7db462762c484f038f37273a38dfdc78 2024-11-19T15:16:02,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:02,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:02,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:02,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:02,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:02,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/9683070b55634c48b8ca39b8dfc1350d is 50, key is test_row_0/C:col10/1732029361713/Put/seqid=0 2024-11-19T15:16:02,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742341_1517 (size=12301) 2024-11-19T15:16:02,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/9683070b55634c48b8ca39b8dfc1350d 2024-11-19T15:16:02,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/7daf1f095b394083ab46688fb26a4d90 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/7daf1f095b394083ab46688fb26a4d90 2024-11-19T15:16:02,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/7daf1f095b394083ab46688fb26a4d90, entries=200, sequenceid=303, filesize=14.4 K 2024-11-19T15:16:02,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/7db462762c484f038f37273a38dfdc78 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7db462762c484f038f37273a38dfdc78 2024-11-19T15:16:02,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7db462762c484f038f37273a38dfdc78, entries=150, sequenceid=303, filesize=12.0 K 2024-11-19T15:16:02,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/9683070b55634c48b8ca39b8dfc1350d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/9683070b55634c48b8ca39b8dfc1350d 2024-11-19T15:16:02,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/9683070b55634c48b8ca39b8dfc1350d, entries=150, sequenceid=303, filesize=12.0 K 2024-11-19T15:16:02,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 9d89d22f9c07cea236aa6c26f89b6b4f in 904ms, sequenceid=303, compaction requested=false 2024-11-19T15:16:02,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:02,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T15:16:02,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:02,739 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T15:16:02,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:16:02,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:02,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:16:02,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:02,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:16:02,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/2d1191b1cc7543e7ad103cffc8e473aa is 50, key is test_row_0/A:col10/1732029361778/Put/seqid=0 2024-11-19T15:16:02,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742342_1518 (size=12301) 2024-11-19T15:16:02,748 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/2d1191b1cc7543e7ad103cffc8e473aa 2024-11-19T15:16:02,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/d92d8d6c9aa14ff7835d6cb78dc154d3 is 50, key is test_row_0/B:col10/1732029361778/Put/seqid=0 2024-11-19T15:16:02,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742343_1519 (size=12301) 2024-11-19T15:16:02,759 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/d92d8d6c9aa14ff7835d6cb78dc154d3 2024-11-19T15:16:02,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/5c0ca47b5be94f8fb6ae108abce55fe3 is 50, key is test_row_0/C:col10/1732029361778/Put/seqid=0 2024-11-19T15:16:02,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742344_1520 (size=12301) 2024-11-19T15:16:02,769 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/5c0ca47b5be94f8fb6ae108abce55fe3 2024-11-19T15:16:02,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/2d1191b1cc7543e7ad103cffc8e473aa as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2d1191b1cc7543e7ad103cffc8e473aa 2024-11-19T15:16:02,777 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2d1191b1cc7543e7ad103cffc8e473aa, entries=150, sequenceid=315, filesize=12.0 K 2024-11-19T15:16:02,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/d92d8d6c9aa14ff7835d6cb78dc154d3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/d92d8d6c9aa14ff7835d6cb78dc154d3 2024-11-19T15:16:02,781 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/d92d8d6c9aa14ff7835d6cb78dc154d3, entries=150, sequenceid=315, filesize=12.0 K 2024-11-19T15:16:02,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/5c0ca47b5be94f8fb6ae108abce55fe3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/5c0ca47b5be94f8fb6ae108abce55fe3 2024-11-19T15:16:02,785 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/5c0ca47b5be94f8fb6ae108abce55fe3, entries=150, sequenceid=315, filesize=12.0 K 2024-11-19T15:16:02,786 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 9d89d22f9c07cea236aa6c26f89b6b4f in 47ms, sequenceid=315, compaction requested=true 2024-11-19T15:16:02,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:02,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:02,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-19T15:16:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-19T15:16:02,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-19T15:16:02,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3460 sec 2024-11-19T15:16:02,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.3540 sec 2024-11-19T15:16:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:16:02,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:16:02,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:16:02,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:02,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:16:02,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:02,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:16:02,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:02,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/57df6938fbf64cb4aea94dd55e38f2ce is 50, key is test_row_0/A:col10/1732029362809/Put/seqid=0 2024-11-19T15:16:02,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742345_1521 (size=17181) 2024-11-19T15:16:02,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:02,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029362840, max timestamp=1732029362877Num uniques 859; 2024-11-19T15:16:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029422866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:02,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029362840, max timestamp=1732029362877Num uniques 859; 2024-11-19T15:16:02,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:16:02,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:16:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:02,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029362840, max timestamp=1732029362877Num uniques 859; 2024-11-19T15:16:02,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:16:02,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029422970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:02,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029422972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:02,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029422972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:02,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029422974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029422984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:03,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029423084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:03,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029423094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029423094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:03,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029423194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:03,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/57df6938fbf64cb4aea94dd55e38f2ce 2024-11-19T15:16:03,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/767fd44f579b46c583d56ff00ea27055 is 50, key is test_row_0/B:col10/1732029362809/Put/seqid=0 2024-11-19T15:16:03,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742346_1522 (size=12301) 2024-11-19T15:16:03,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029423304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:03,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029423314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:03,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029423314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:03,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029423514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:03,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029423624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:03,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029423634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:03,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029423634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:03,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/767fd44f579b46c583d56ff00ea27055 2024-11-19T15:16:03,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/f1d25d002bf64f10a7c9ac630c72c62f is 50, key is test_row_0/C:col10/1732029362809/Put/seqid=0 2024-11-19T15:16:03,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742347_1523 (size=12301) 2024-11-19T15:16:03,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/f1d25d002bf64f10a7c9ac630c72c62f 2024-11-19T15:16:03,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/57df6938fbf64cb4aea94dd55e38f2ce as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/57df6938fbf64cb4aea94dd55e38f2ce 2024-11-19T15:16:03,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/57df6938fbf64cb4aea94dd55e38f2ce, entries=250, sequenceid=325, filesize=16.8 K 2024-11-19T15:16:03,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/767fd44f579b46c583d56ff00ea27055 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/767fd44f579b46c583d56ff00ea27055 2024-11-19T15:16:03,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/767fd44f579b46c583d56ff00ea27055, entries=150, sequenceid=325, filesize=12.0 K 2024-11-19T15:16:03,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/f1d25d002bf64f10a7c9ac630c72c62f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/f1d25d002bf64f10a7c9ac630c72c62f 2024-11-19T15:16:03,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/f1d25d002bf64f10a7c9ac630c72c62f, entries=150, sequenceid=325, filesize=12.0 K 2024-11-19T15:16:03,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 9d89d22f9c07cea236aa6c26f89b6b4f in 862ms, sequenceid=325, compaction requested=true 2024-11-19T15:16:03,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:03,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:16:03,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:03,674 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:16:03,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:16:03,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:03,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:16:03,674 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:16:03,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:03,674 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57240 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:16:03,675 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/A is initiating minor compaction (all files) 2024-11-19T15:16:03,675 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:16:03,675 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/A in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:03,675 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/89f68965dac44ec9b1ee723863bf5b91, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/7daf1f095b394083ab46688fb26a4d90, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2d1191b1cc7543e7ad103cffc8e473aa, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/57df6938fbf64cb4aea94dd55e38f2ce] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=55.9 K 2024-11-19T15:16:03,675 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/B is initiating minor compaction (all files) 2024-11-19T15:16:03,675 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/B in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:03,675 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/496c54d041314f56a776ec0958dad3cc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7db462762c484f038f37273a38dfdc78, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/d92d8d6c9aa14ff7835d6cb78dc154d3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/767fd44f579b46c583d56ff00ea27055] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=48.8 K 2024-11-19T15:16:03,675 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89f68965dac44ec9b1ee723863bf5b91, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732029360403 2024-11-19T15:16:03,676 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 496c54d041314f56a776ec0958dad3cc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732029360403 2024-11-19T15:16:03,676 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7daf1f095b394083ab46688fb26a4d90, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732029360583 2024-11-19T15:16:03,676 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7db462762c484f038f37273a38dfdc78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732029360583 2024-11-19T15:16:03,676 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d1191b1cc7543e7ad103cffc8e473aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732029361744 2024-11-19T15:16:03,676 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d92d8d6c9aa14ff7835d6cb78dc154d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732029361744 2024-11-19T15:16:03,677 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57df6938fbf64cb4aea94dd55e38f2ce, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732029362796 2024-11-19T15:16:03,677 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 767fd44f579b46c583d56ff00ea27055, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732029362800 2024-11-19T15:16:03,685 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#A#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:03,686 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/dffea228178d4a91b793b0a074afdbc5 is 50, key is test_row_0/A:col10/1732029362809/Put/seqid=0 2024-11-19T15:16:03,687 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#B#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:03,688 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/8353fb3884ce424986b5f94620f1e8ac is 50, key is test_row_0/B:col10/1732029362809/Put/seqid=0 2024-11-19T15:16:03,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742348_1524 (size=13153) 2024-11-19T15:16:03,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742349_1525 (size=13153) 2024-11-19T15:16:03,695 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/dffea228178d4a91b793b0a074afdbc5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/dffea228178d4a91b793b0a074afdbc5 2024-11-19T15:16:03,698 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/A of 9d89d22f9c07cea236aa6c26f89b6b4f into dffea228178d4a91b793b0a074afdbc5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:03,699 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:03,699 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/A, priority=12, startTime=1732029363673; duration=0sec 2024-11-19T15:16:03,699 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:03,699 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:A 2024-11-19T15:16:03,699 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:16:03,700 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:16:03,701 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/C is initiating minor compaction (all files) 2024-11-19T15:16:03,701 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/C in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:03,701 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/78ecee4865b14ce49bc58eb461162e20, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/9683070b55634c48b8ca39b8dfc1350d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/5c0ca47b5be94f8fb6ae108abce55fe3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/f1d25d002bf64f10a7c9ac630c72c62f] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=48.8 K 2024-11-19T15:16:03,701 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78ecee4865b14ce49bc58eb461162e20, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732029360403 2024-11-19T15:16:03,701 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9683070b55634c48b8ca39b8dfc1350d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732029360583 2024-11-19T15:16:03,702 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c0ca47b5be94f8fb6ae108abce55fe3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732029361744 2024-11-19T15:16:03,702 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1d25d002bf64f10a7c9ac630c72c62f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732029362800 2024-11-19T15:16:03,709 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#C#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:03,709 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/f3e96979c1424d34b17887d86514d66a is 50, key is test_row_0/C:col10/1732029362809/Put/seqid=0 2024-11-19T15:16:03,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742350_1526 (size=13153) 2024-11-19T15:16:03,724 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/f3e96979c1424d34b17887d86514d66a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/f3e96979c1424d34b17887d86514d66a 2024-11-19T15:16:03,729 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/C of 9d89d22f9c07cea236aa6c26f89b6b4f into f3e96979c1424d34b17887d86514d66a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:03,729 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:03,730 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/C, priority=12, startTime=1732029363674; duration=0sec 2024-11-19T15:16:03,730 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:03,730 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:C 2024-11-19T15:16:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:16:04,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-19T15:16:04,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:16:04,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T15:16:04,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:16:04,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T15:16:04,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:16:04,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T15:16:04,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/8ceafa0f5ef2421a93212ed302dca728 is 50, key is test_row_0/A:col10/1732029362970/Put/seqid=0 2024-11-19T15:16:04,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742351_1527 (size=14741) 2024-11-19T15:16:04,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029424042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:04,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029424050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:04,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:04,098 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/8353fb3884ce424986b5f94620f1e8ac as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/8353fb3884ce424986b5f94620f1e8ac 2024-11-19T15:16:04,102 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/B of 9d89d22f9c07cea236aa6c26f89b6b4f into 8353fb3884ce424986b5f94620f1e8ac(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:04,102 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:04,102 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/B, priority=12, startTime=1732029363674; duration=0sec 2024-11-19T15:16:04,102 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:04,102 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:B 2024-11-19T15:16:04,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029424144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:04,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029424144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:04,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029424144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029424164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029424164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:04,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029424384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:04,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029424384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:04,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/8ceafa0f5ef2421a93212ed302dca728 2024-11-19T15:16:04,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/15c335bccb914019bfe6ea5198253e55 is 50, key is test_row_0/B:col10/1732029362970/Put/seqid=0 2024-11-19T15:16:04,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742352_1528 (size=12301) 2024-11-19T15:16:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T15:16:04,605 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:04,605 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:16:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-19T15:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T15:16:04,607 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:16:04,608 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:16:04,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:16:04,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029424704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:04,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029424704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T15:16:04,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T15:16:04,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:04,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:04,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:04,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:04,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:04,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:04,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/15c335bccb914019bfe6ea5198253e55 2024-11-19T15:16:04,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/285ed1217c304d0991f51627d8d45c97 is 50, key is test_row_0/C:col10/1732029362970/Put/seqid=0 2024-11-19T15:16:04,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742353_1529 (size=12301) 2024-11-19T15:16:04,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T15:16:04,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:04,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:04,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:04,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:04,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:04,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T15:16:05,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T15:16:05,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:05,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:05,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:05,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:05,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:05,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029425164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:05,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029425164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:05,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029425164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:05,175 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 9d89d22f9c07cea236aa6c26f89b6b4f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:16:05,175 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in 9d89d22f9c07cea236aa6c26f89b6b4f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:16:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:05,176 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 9d89d22f9c07cea236aa6c26f89b6b4f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:16:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:05,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T15:16:05,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:05,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:05,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029425224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:05,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029425224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T15:16:05,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/285ed1217c304d0991f51627d8d45c97 2024-11-19T15:16:05,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/8ceafa0f5ef2421a93212ed302dca728 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/8ceafa0f5ef2421a93212ed302dca728 2024-11-19T15:16:05,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/8ceafa0f5ef2421a93212ed302dca728, entries=200, sequenceid=365, filesize=14.4 K 2024-11-19T15:16:05,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/15c335bccb914019bfe6ea5198253e55 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/15c335bccb914019bfe6ea5198253e55 2024-11-19T15:16:05,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/15c335bccb914019bfe6ea5198253e55, entries=150, sequenceid=365, filesize=12.0 K 2024-11-19T15:16:05,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/285ed1217c304d0991f51627d8d45c97 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/285ed1217c304d0991f51627d8d45c97 2024-11-19T15:16:05,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/285ed1217c304d0991f51627d8d45c97, entries=150, sequenceid=365, filesize=12.0 K 2024-11-19T15:16:05,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 9d89d22f9c07cea236aa6c26f89b6b4f in 1253ms, sequenceid=365, compaction requested=false 2024-11-19T15:16:05,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:05,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T15:16:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:05,372 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T15:16:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:16:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:16:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:16:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:05,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/92d71e05b74d4a55b542394ab78aca58 is 50, key is test_row_0/A:col10/1732029364049/Put/seqid=0 2024-11-19T15:16:05,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742354_1530 (size=9857) 2024-11-19T15:16:05,384 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/92d71e05b74d4a55b542394ab78aca58 2024-11-19T15:16:05,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/d0c3ef920661408db225f49ab50b32dc is 50, key is test_row_0/B:col10/1732029364049/Put/seqid=0 2024-11-19T15:16:05,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742355_1531 (size=9857) 2024-11-19T15:16:05,396 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/d0c3ef920661408db225f49ab50b32dc 2024-11-19T15:16:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/b63fb89babc542038747285a544d711f is 50, key is test_row_0/C:col10/1732029364049/Put/seqid=0 2024-11-19T15:16:05,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742356_1532 (size=9857) 2024-11-19T15:16:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T15:16:05,806 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/b63fb89babc542038747285a544d711f 2024-11-19T15:16:05,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/92d71e05b74d4a55b542394ab78aca58 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/92d71e05b74d4a55b542394ab78aca58 2024-11-19T15:16:05,814 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/92d71e05b74d4a55b542394ab78aca58, entries=100, sequenceid=374, filesize=9.6 K 2024-11-19T15:16:05,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/d0c3ef920661408db225f49ab50b32dc as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/d0c3ef920661408db225f49ab50b32dc 2024-11-19T15:16:05,818 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/d0c3ef920661408db225f49ab50b32dc, entries=100, sequenceid=374, filesize=9.6 K 2024-11-19T15:16:05,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/b63fb89babc542038747285a544d711f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/b63fb89babc542038747285a544d711f 2024-11-19T15:16:05,824 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/b63fb89babc542038747285a544d711f, entries=100, sequenceid=374, filesize=9.6 K 2024-11-19T15:16:05,825 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9d89d22f9c07cea236aa6c26f89b6b4f in 453ms, sequenceid=374, compaction requested=true 2024-11-19T15:16:05,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:05,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:05,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-19T15:16:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-19T15:16:05,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-19T15:16:05,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2190 sec 2024-11-19T15:16:05,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.2240 sec 2024-11-19T15:16:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:16:06,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:16:06,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:16:06,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:06,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:16:06,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:06,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:16:06,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:06,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/9fc21742aa1a4cb69fed9d7d1cb6abff is 50, key is test_row_0/A:col10/1732029366254/Put/seqid=0 2024-11-19T15:16:06,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742357_1533 (size=12301) 2024-11-19T15:16:06,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029366289, max timestamp=1732029366346Num uniques 859; 2024-11-19T15:16:06,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029366289, max timestamp=1732029366346Num uniques 859; 2024-11-19T15:16:06,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:16:06,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:16:06,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029366289, max timestamp=1732029366346Num uniques 859; 2024-11-19T15:16:06,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:16:06,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029426449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:06,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029426454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:06,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029426574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:06,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029426574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:06,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:06,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:06,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:06,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:06,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:06,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:06,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/9fc21742aa1a4cb69fed9d7d1cb6abff 2024-11-19T15:16:06,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/151a5ad0ebf4488aaafbed5b4872dc01 is 50, key is test_row_0/B:col10/1732029366254/Put/seqid=0 2024-11-19T15:16:06,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742358_1534 (size=12301) 2024-11-19T15:16:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T15:16:06,755 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:06,755 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:16:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-19T15:16:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T15:16:06,757 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:16:06,757 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:16:06,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:16:06,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029426783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:06,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029426784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T15:16:06,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T15:16:06,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:06,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:06,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:06,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:06,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:06,980 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T15:16:07,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T15:16:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:07,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:07,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T15:16:07,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/151a5ad0ebf4488aaafbed5b4872dc01 2024-11-19T15:16:07,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/680f7d7f88bc4c4fbd3fc38f7bfe27b1 is 50, key is test_row_0/C:col10/1732029366254/Put/seqid=0 2024-11-19T15:16:07,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742359_1535 (size=12301) 2024-11-19T15:16:07,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54904 deadline: 1732029427104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:07,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54870 deadline: 1732029427104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:07,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1732029427194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:07,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:07,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1732029427194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:07,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:07,203 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 9d89d22f9c07cea236aa6c26f89b6b4f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:16:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1732029427194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:07,204 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in 9d89d22f9c07cea236aa6c26f89b6b4f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:16:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:07,204 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_2' in 9d89d22f9c07cea236aa6c26f89b6b4f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:16:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 , the old value is region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., hostname=50570b5b74e7,46557,1732029218639, seqNum=2 because the exception is null or not the one we care about 2024-11-19T15:16:07,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T15:16:07,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:07,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:07,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:07,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:07,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:07,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T15:16:07,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:07,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:07,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:07,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:07,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T15:16:07,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/680f7d7f88bc4c4fbd3fc38f7bfe27b1 2024-11-19T15:16:07,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/9fc21742aa1a4cb69fed9d7d1cb6abff as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/9fc21742aa1a4cb69fed9d7d1cb6abff 2024-11-19T15:16:07,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/9fc21742aa1a4cb69fed9d7d1cb6abff, entries=150, sequenceid=384, filesize=12.0 K 2024-11-19T15:16:07,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/151a5ad0ebf4488aaafbed5b4872dc01 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/151a5ad0ebf4488aaafbed5b4872dc01 2024-11-19T15:16:07,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/151a5ad0ebf4488aaafbed5b4872dc01, entries=150, sequenceid=384, filesize=12.0 K 2024-11-19T15:16:07,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/680f7d7f88bc4c4fbd3fc38f7bfe27b1 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/680f7d7f88bc4c4fbd3fc38f7bfe27b1 2024-11-19T15:16:07,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/680f7d7f88bc4c4fbd3fc38f7bfe27b1, entries=150, sequenceid=384, filesize=12.0 K 2024-11-19T15:16:07,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9d89d22f9c07cea236aa6c26f89b6b4f in 1249ms, sequenceid=384, compaction requested=true 2024-11-19T15:16:07,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:07,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:16:07,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:07,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:16:07,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:07,504 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:16:07,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d89d22f9c07cea236aa6c26f89b6b4f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:16:07,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:07,504 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:16:07,505 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:16:07,505 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50052 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:16:07,505 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/B is initiating minor compaction (all files) 2024-11-19T15:16:07,505 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/A is initiating minor compaction (all files) 2024-11-19T15:16:07,506 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/B in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:07,506 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/A in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:07,506 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/8353fb3884ce424986b5f94620f1e8ac, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/15c335bccb914019bfe6ea5198253e55, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/d0c3ef920661408db225f49ab50b32dc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/151a5ad0ebf4488aaafbed5b4872dc01] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=46.5 K 2024-11-19T15:16:07,506 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/dffea228178d4a91b793b0a074afdbc5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/8ceafa0f5ef2421a93212ed302dca728, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/92d71e05b74d4a55b542394ab78aca58, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/9fc21742aa1a4cb69fed9d7d1cb6abff] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=48.9 K 2024-11-19T15:16:07,506 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 8353fb3884ce424986b5f94620f1e8ac, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732029362800 2024-11-19T15:16:07,506 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting dffea228178d4a91b793b0a074afdbc5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732029362800 2024-11-19T15:16:07,506 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ceafa0f5ef2421a93212ed302dca728, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732029362952 2024-11-19T15:16:07,506 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c335bccb914019bfe6ea5198253e55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732029362969 2024-11-19T15:16:07,507 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92d71e05b74d4a55b542394ab78aca58, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732029364040 2024-11-19T15:16:07,507 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d0c3ef920661408db225f49ab50b32dc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732029364040 2024-11-19T15:16:07,507 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fc21742aa1a4cb69fed9d7d1cb6abff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732029366248 2024-11-19T15:16:07,507 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 151a5ad0ebf4488aaafbed5b4872dc01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732029366248 2024-11-19T15:16:07,516 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#A#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:07,516 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#B#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:07,517 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/1180f6b72a00444c9ec073d6277809c3 is 50, key is test_row_0/B:col10/1732029366254/Put/seqid=0 2024-11-19T15:16:07,517 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/b71dff15354040bc85b5899c9c234251 is 50, key is test_row_0/A:col10/1732029366254/Put/seqid=0 2024-11-19T15:16:07,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T15:16:07,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:07,521 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T15:16:07,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:16:07,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:07,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:16:07,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:07,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:16:07,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:07,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742360_1536 (size=13289) 2024-11-19T15:16:07,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742361_1537 (size=13289) 2024-11-19T15:16:07,528 INFO [Thread-2032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2032. 2024-11-19T15:16:07,528 DEBUG [Thread-2032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:07,528 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,528 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/238502d2a75744f4a2f9e5b2d9729652 is 50, key is test_row_0/A:col10/1732029366448/Put/seqid=0 2024-11-19T15:16:07,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:07,532 INFO [Thread-2028 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2028. 2024-11-19T15:16:07,532 DEBUG [Thread-2028 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:07,532 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,532 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,532 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:07,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742362_1538 (size=12301) 2024-11-19T15:16:07,536 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/238502d2a75744f4a2f9e5b2d9729652 2024-11-19T15:16:07,539 INFO [Thread-2024 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2024. 2024-11-19T15:16:07,539 DEBUG [Thread-2024 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:07,539 DEBUG [Thread-2024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,539 DEBUG [Thread-2024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:07,541 INFO [Thread-2030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2030. 2024-11-19T15:16:07,541 DEBUG [Thread-2030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:07,541 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,541 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,541 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:07,542 INFO [Thread-2026 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2026. 2024-11-19T15:16:07,542 DEBUG [Thread-2026 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:07,542 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,543 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,543 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:07,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/026f5d05dc064c87a8a6c13e4536e3e9 is 50, key is test_row_0/B:col10/1732029366448/Put/seqid=0 2024-11-19T15:16:07,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742363_1539 (size=12301) 2024-11-19T15:16:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:16:07,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. as already flushing 2024-11-19T15:16:07,615 INFO [Thread-2019 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2019. 2024-11-19T15:16:07,615 DEBUG [Thread-2019 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:07,615 INFO [Thread-2017 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2017. 2024-11-19T15:16:07,615 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,615 DEBUG [Thread-2017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:07,615 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,615 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:07,615 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:07,616 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:07,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T15:16:07,927 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/1180f6b72a00444c9ec073d6277809c3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/1180f6b72a00444c9ec073d6277809c3 2024-11-19T15:16:07,929 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/b71dff15354040bc85b5899c9c234251 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/b71dff15354040bc85b5899c9c234251 2024-11-19T15:16:07,931 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/B of 9d89d22f9c07cea236aa6c26f89b6b4f into 1180f6b72a00444c9ec073d6277809c3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:07,931 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:07,931 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/B, priority=12, startTime=1732029367504; duration=0sec 2024-11-19T15:16:07,931 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:07,931 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:B 2024-11-19T15:16:07,931 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:16:07,932 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:16:07,932 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 9d89d22f9c07cea236aa6c26f89b6b4f/C is initiating minor compaction (all files) 2024-11-19T15:16:07,932 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d89d22f9c07cea236aa6c26f89b6b4f/C in TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:07,932 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/f3e96979c1424d34b17887d86514d66a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/285ed1217c304d0991f51627d8d45c97, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/b63fb89babc542038747285a544d711f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/680f7d7f88bc4c4fbd3fc38f7bfe27b1] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp, totalSize=46.5 K 2024-11-19T15:16:07,933 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting f3e96979c1424d34b17887d86514d66a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732029362800 2024-11-19T15:16:07,933 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/A of 9d89d22f9c07cea236aa6c26f89b6b4f into b71dff15354040bc85b5899c9c234251(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:07,933 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:07,933 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/A, priority=12, startTime=1732029367504; duration=0sec 2024-11-19T15:16:07,933 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 285ed1217c304d0991f51627d8d45c97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732029362969 2024-11-19T15:16:07,933 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:07,933 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:A 2024-11-19T15:16:07,933 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting b63fb89babc542038747285a544d711f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732029364040 2024-11-19T15:16:07,933 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 680f7d7f88bc4c4fbd3fc38f7bfe27b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732029366248 2024-11-19T15:16:07,938 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d89d22f9c07cea236aa6c26f89b6b4f#C#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:07,939 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/ab927eed683045078437b6cc86c2dd5d is 50, key is test_row_0/C:col10/1732029366254/Put/seqid=0 2024-11-19T15:16:07,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742364_1540 (size=13289) 2024-11-19T15:16:07,946 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/026f5d05dc064c87a8a6c13e4536e3e9 2024-11-19T15:16:07,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/4072645226ad44ce94277fe69aab2681 is 50, key is test_row_0/C:col10/1732029366448/Put/seqid=0 2024-11-19T15:16:07,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742365_1541 (size=12301) 2024-11-19T15:16:08,346 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/ab927eed683045078437b6cc86c2dd5d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/ab927eed683045078437b6cc86c2dd5d 2024-11-19T15:16:08,349 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d89d22f9c07cea236aa6c26f89b6b4f/C of 9d89d22f9c07cea236aa6c26f89b6b4f into ab927eed683045078437b6cc86c2dd5d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:08,349 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:08,349 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., storeName=9d89d22f9c07cea236aa6c26f89b6b4f/C, priority=12, startTime=1732029367504; duration=0sec 2024-11-19T15:16:08,349 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:08,349 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d89d22f9c07cea236aa6c26f89b6b4f:C 2024-11-19T15:16:08,354 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/4072645226ad44ce94277fe69aab2681 2024-11-19T15:16:08,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/238502d2a75744f4a2f9e5b2d9729652 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/238502d2a75744f4a2f9e5b2d9729652 2024-11-19T15:16:08,360 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/238502d2a75744f4a2f9e5b2d9729652, entries=150, sequenceid=421, filesize=12.0 K 2024-11-19T15:16:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/026f5d05dc064c87a8a6c13e4536e3e9 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/026f5d05dc064c87a8a6c13e4536e3e9 2024-11-19T15:16:08,363 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/026f5d05dc064c87a8a6c13e4536e3e9, entries=150, sequenceid=421, filesize=12.0 K 2024-11-19T15:16:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/4072645226ad44ce94277fe69aab2681 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/4072645226ad44ce94277fe69aab2681 2024-11-19T15:16:08,366 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/4072645226ad44ce94277fe69aab2681, entries=150, sequenceid=421, filesize=12.0 K 2024-11-19T15:16:08,367 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=13.42 KB/13740 for 9d89d22f9c07cea236aa6c26f89b6b4f in 846ms, sequenceid=421, compaction requested=false 2024-11-19T15:16:08,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 9d89d22f9c07cea236aa6c26f89b6b4f: 2024-11-19T15:16:08,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:08,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-19T15:16:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-19T15:16:08,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-19T15:16:08,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6120 sec 2024-11-19T15:16:08,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.6160 sec 2024-11-19T15:16:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T15:16:08,905 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:11,235 INFO [Thread-2021 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2021. 2024-11-19T15:16:11,235 DEBUG [Thread-2021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:11,235 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:11,235 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:11,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:11,245 INFO [Thread-2013 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2013. 2024-11-19T15:16:11,245 DEBUG [Thread-2013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:11,245 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:11,245 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:11,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:11,255 INFO [Thread-2015 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2015. 2024-11-19T15:16:11,255 DEBUG [Thread-2015 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:11,255 DEBUG [Thread-2015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:11,255 DEBUG [Thread-2015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:11,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1080 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3237 rows 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1080 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3237 rows 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1084 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3251 rows 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1078 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3231 rows 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1086 2024-11-19T15:16:11,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3255 rows 2024-11-19T15:16:11,256 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T15:16:11,256 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T15:16:11,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:11,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:11,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:11,257 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T15:16:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T15:16:11,259 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029371259"}]},"ts":"1732029371259"} 2024-11-19T15:16:11,260 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T15:16:11,260 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T15:16:11,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T15:16:11,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d89d22f9c07cea236aa6c26f89b6b4f, UNASSIGN}] 2024-11-19T15:16:11,262 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d89d22f9c07cea236aa6c26f89b6b4f, UNASSIGN 2024-11-19T15:16:11,263 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=9d89d22f9c07cea236aa6c26f89b6b4f, regionState=CLOSING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:11,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d89d22f9c07cea236aa6c26f89b6b4f, UNASSIGN because future has completed 2024-11-19T15:16:11,264 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T15:16:11,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:16:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T15:16:11,416 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:16:11,416 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T15:16:11,416 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 9d89d22f9c07cea236aa6c26f89b6b4f, disabling compactions & flushes 2024-11-19T15:16:11,416 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:11,416 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:11,416 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. after waiting 0 ms 2024-11-19T15:16:11,416 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:11,416 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 9d89d22f9c07cea236aa6c26f89b6b4f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T15:16:11,416 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=A 2024-11-19T15:16:11,417 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:11,417 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=B 2024-11-19T15:16:11,417 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:11,417 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9d89d22f9c07cea236aa6c26f89b6b4f, store=C 2024-11-19T15:16:11,417 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:11,420 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/66ef0e1f5c3d4ca89837034a01b00045 is 50, key is test_row_0/A:col10/1732029371244/Put/seqid=0 2024-11-19T15:16:11,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742366_1542 (size=12301) 2024-11-19T15:16:11,423 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/66ef0e1f5c3d4ca89837034a01b00045 2024-11-19T15:16:11,430 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/11a1978f07ba46899f26f6a9e060e76e is 50, key is test_row_0/B:col10/1732029371244/Put/seqid=0 2024-11-19T15:16:11,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742367_1543 (size=12301) 2024-11-19T15:16:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T15:16:11,833 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/11a1978f07ba46899f26f6a9e060e76e 2024-11-19T15:16:11,838 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/4d7ed1ff09b343d9852358b837bf3190 is 50, key is test_row_0/C:col10/1732029371244/Put/seqid=0 2024-11-19T15:16:11,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742368_1544 (size=12301) 2024-11-19T15:16:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T15:16:12,242 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/4d7ed1ff09b343d9852358b837bf3190 2024-11-19T15:16:12,245 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/A/66ef0e1f5c3d4ca89837034a01b00045 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/66ef0e1f5c3d4ca89837034a01b00045 2024-11-19T15:16:12,248 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/66ef0e1f5c3d4ca89837034a01b00045, entries=150, sequenceid=432, filesize=12.0 K 2024-11-19T15:16:12,249 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/B/11a1978f07ba46899f26f6a9e060e76e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/11a1978f07ba46899f26f6a9e060e76e 2024-11-19T15:16:12,251 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/11a1978f07ba46899f26f6a9e060e76e, entries=150, sequenceid=432, filesize=12.0 K 2024-11-19T15:16:12,252 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/.tmp/C/4d7ed1ff09b343d9852358b837bf3190 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/4d7ed1ff09b343d9852358b837bf3190 2024-11-19T15:16:12,255 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/4d7ed1ff09b343d9852358b837bf3190, entries=150, sequenceid=432, filesize=12.0 K 2024-11-19T15:16:12,256 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9d89d22f9c07cea236aa6c26f89b6b4f in 839ms, sequenceid=432, compaction requested=true 2024-11-19T15:16:12,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/1e70096196b34112b8a0f301bea06958, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/cabc2c11c79d434c810ce5cf1e4c0eed, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/4f41d71caece48fbbf711b9f75d536ef, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/099809012e43451e9380fbd6f533051c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/af311936d1964281bfd998a0883eb3a0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/f3b73a9a21014f5ca40a44056c93befd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/73f4022f820b48f38cb5df8b9070b2fd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/57cc6699130c4b52b0f7f29438a3ce54, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/b49aa668766a47c08669571047ee4da5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/e4f34fd2df0f4bab85f0790d1aad1383, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2745b0661e1945b48ab2f66879994a19, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/97e5b37b4bb04bb2bf4aa3cfd0c77437, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/91ecdd8aa12a459d9d6a4dca761e528c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/66dbba890d1745a9bc63379e67326195, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/ac57fbf280414f0d834cbd34f3688cca, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/89c17910005940faab5a3b19c63cf816, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2414150219d04f3baecce42c1d17878e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/c65e30b44abc4ec38c605ac874bc0522, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/9c3ccef1bd2543c29aaffc5208a008b8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/5879bb5db5f040fa89a2f2a219743c46, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/89f68965dac44ec9b1ee723863bf5b91, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/22ba5b8f28e84026a708cb85628de02f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/7daf1f095b394083ab46688fb26a4d90, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2d1191b1cc7543e7ad103cffc8e473aa, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/57df6938fbf64cb4aea94dd55e38f2ce, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/dffea228178d4a91b793b0a074afdbc5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/8ceafa0f5ef2421a93212ed302dca728, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/92d71e05b74d4a55b542394ab78aca58, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/9fc21742aa1a4cb69fed9d7d1cb6abff] to archive 2024-11-19T15:16:12,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:16:12,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/1e70096196b34112b8a0f301bea06958 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/1e70096196b34112b8a0f301bea06958 2024-11-19T15:16:12,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/cabc2c11c79d434c810ce5cf1e4c0eed to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/cabc2c11c79d434c810ce5cf1e4c0eed 2024-11-19T15:16:12,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/4f41d71caece48fbbf711b9f75d536ef to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/4f41d71caece48fbbf711b9f75d536ef 2024-11-19T15:16:12,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/099809012e43451e9380fbd6f533051c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/099809012e43451e9380fbd6f533051c 2024-11-19T15:16:12,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/af311936d1964281bfd998a0883eb3a0 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/af311936d1964281bfd998a0883eb3a0 2024-11-19T15:16:12,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/f3b73a9a21014f5ca40a44056c93befd to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/f3b73a9a21014f5ca40a44056c93befd 2024-11-19T15:16:12,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/73f4022f820b48f38cb5df8b9070b2fd to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/73f4022f820b48f38cb5df8b9070b2fd 2024-11-19T15:16:12,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/57cc6699130c4b52b0f7f29438a3ce54 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/57cc6699130c4b52b0f7f29438a3ce54 2024-11-19T15:16:12,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/b49aa668766a47c08669571047ee4da5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/b49aa668766a47c08669571047ee4da5 2024-11-19T15:16:12,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/e4f34fd2df0f4bab85f0790d1aad1383 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/e4f34fd2df0f4bab85f0790d1aad1383 2024-11-19T15:16:12,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2745b0661e1945b48ab2f66879994a19 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2745b0661e1945b48ab2f66879994a19 2024-11-19T15:16:12,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/97e5b37b4bb04bb2bf4aa3cfd0c77437 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/97e5b37b4bb04bb2bf4aa3cfd0c77437 2024-11-19T15:16:12,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/91ecdd8aa12a459d9d6a4dca761e528c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/91ecdd8aa12a459d9d6a4dca761e528c 2024-11-19T15:16:12,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/66dbba890d1745a9bc63379e67326195 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/66dbba890d1745a9bc63379e67326195 2024-11-19T15:16:12,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/ac57fbf280414f0d834cbd34f3688cca to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/ac57fbf280414f0d834cbd34f3688cca 2024-11-19T15:16:12,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/89c17910005940faab5a3b19c63cf816 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/89c17910005940faab5a3b19c63cf816 2024-11-19T15:16:12,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2414150219d04f3baecce42c1d17878e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2414150219d04f3baecce42c1d17878e 2024-11-19T15:16:12,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/c65e30b44abc4ec38c605ac874bc0522 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/c65e30b44abc4ec38c605ac874bc0522 2024-11-19T15:16:12,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/9c3ccef1bd2543c29aaffc5208a008b8 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/9c3ccef1bd2543c29aaffc5208a008b8 2024-11-19T15:16:12,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/5879bb5db5f040fa89a2f2a219743c46 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/5879bb5db5f040fa89a2f2a219743c46 2024-11-19T15:16:12,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/89f68965dac44ec9b1ee723863bf5b91 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/89f68965dac44ec9b1ee723863bf5b91 2024-11-19T15:16:12,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/22ba5b8f28e84026a708cb85628de02f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/22ba5b8f28e84026a708cb85628de02f 2024-11-19T15:16:12,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/7daf1f095b394083ab46688fb26a4d90 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/7daf1f095b394083ab46688fb26a4d90 2024-11-19T15:16:12,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2d1191b1cc7543e7ad103cffc8e473aa to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/2d1191b1cc7543e7ad103cffc8e473aa 2024-11-19T15:16:12,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/57df6938fbf64cb4aea94dd55e38f2ce to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/57df6938fbf64cb4aea94dd55e38f2ce 2024-11-19T15:16:12,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/dffea228178d4a91b793b0a074afdbc5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/dffea228178d4a91b793b0a074afdbc5 2024-11-19T15:16:12,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/8ceafa0f5ef2421a93212ed302dca728 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/8ceafa0f5ef2421a93212ed302dca728 2024-11-19T15:16:12,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/92d71e05b74d4a55b542394ab78aca58 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/92d71e05b74d4a55b542394ab78aca58 2024-11-19T15:16:12,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/9fc21742aa1a4cb69fed9d7d1cb6abff to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/9fc21742aa1a4cb69fed9d7d1cb6abff 2024-11-19T15:16:12,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/71d0ab68b8754b6b8dcb3b75e1f449bb, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/3e66d60aa7a64af5aa96c43ccdded87b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/75eb0879b66b4d45a9ddc3a1d686c872, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/77a77781e0e54f65b49017995b34e2a7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/23292933bac04a45b31bc61c32ecac6d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/25e7accce89f46d5a9cff4beea6cbc1c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/37720bc958e5490fbcb2137dd1cdd19d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/70c8cac92157479494d4d4870e18e2cd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/cd3763cf94e843c29a1000469ba0b182, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/0239a4d81621423784c84593abb28ed5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/0b695affab174d258fc0c038685ff57c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/f629d7fc53a04ca89d35e0c03a9fafae, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/cdd4402b4dd543dd9bb664fb322dfcfb, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/a1f50cb6e433485b8a82beda0ba8edb6, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/8f7f456f45774f19a5cd8657c01215e2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/df3f94736a2e422c8179578b664d7224, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/6430493983294626b558d57c2a2d9793, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7c7dc42b9bf34df8a0756e0dccd1f317, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/69a1227e37c24c42ad66894146c76206, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7ab59a6af08d4309b95c27ceba8ad23f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/496c54d041314f56a776ec0958dad3cc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/2c2f467d65b340319426ce7e1caa9e44, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7db462762c484f038f37273a38dfdc78, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/d92d8d6c9aa14ff7835d6cb78dc154d3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/8353fb3884ce424986b5f94620f1e8ac, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/767fd44f579b46c583d56ff00ea27055, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/15c335bccb914019bfe6ea5198253e55, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/d0c3ef920661408db225f49ab50b32dc, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/151a5ad0ebf4488aaafbed5b4872dc01] to archive 2024-11-19T15:16:12,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:16:12,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/71d0ab68b8754b6b8dcb3b75e1f449bb to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/71d0ab68b8754b6b8dcb3b75e1f449bb 2024-11-19T15:16:12,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/3e66d60aa7a64af5aa96c43ccdded87b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/3e66d60aa7a64af5aa96c43ccdded87b 2024-11-19T15:16:12,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/75eb0879b66b4d45a9ddc3a1d686c872 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/75eb0879b66b4d45a9ddc3a1d686c872 2024-11-19T15:16:12,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/77a77781e0e54f65b49017995b34e2a7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/77a77781e0e54f65b49017995b34e2a7 2024-11-19T15:16:12,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/23292933bac04a45b31bc61c32ecac6d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/23292933bac04a45b31bc61c32ecac6d 2024-11-19T15:16:12,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/25e7accce89f46d5a9cff4beea6cbc1c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/25e7accce89f46d5a9cff4beea6cbc1c 2024-11-19T15:16:12,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/37720bc958e5490fbcb2137dd1cdd19d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/37720bc958e5490fbcb2137dd1cdd19d 2024-11-19T15:16:12,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/70c8cac92157479494d4d4870e18e2cd to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/70c8cac92157479494d4d4870e18e2cd 2024-11-19T15:16:12,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/cd3763cf94e843c29a1000469ba0b182 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/cd3763cf94e843c29a1000469ba0b182 2024-11-19T15:16:12,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/0239a4d81621423784c84593abb28ed5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/0239a4d81621423784c84593abb28ed5 2024-11-19T15:16:12,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/0b695affab174d258fc0c038685ff57c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/0b695affab174d258fc0c038685ff57c 2024-11-19T15:16:12,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/f629d7fc53a04ca89d35e0c03a9fafae to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/f629d7fc53a04ca89d35e0c03a9fafae 2024-11-19T15:16:12,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/cdd4402b4dd543dd9bb664fb322dfcfb to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/cdd4402b4dd543dd9bb664fb322dfcfb 2024-11-19T15:16:12,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/a1f50cb6e433485b8a82beda0ba8edb6 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/a1f50cb6e433485b8a82beda0ba8edb6 2024-11-19T15:16:12,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/8f7f456f45774f19a5cd8657c01215e2 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/8f7f456f45774f19a5cd8657c01215e2 2024-11-19T15:16:12,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/df3f94736a2e422c8179578b664d7224 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/df3f94736a2e422c8179578b664d7224 2024-11-19T15:16:12,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/6430493983294626b558d57c2a2d9793 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/6430493983294626b558d57c2a2d9793 2024-11-19T15:16:12,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7c7dc42b9bf34df8a0756e0dccd1f317 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7c7dc42b9bf34df8a0756e0dccd1f317 2024-11-19T15:16:12,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/69a1227e37c24c42ad66894146c76206 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/69a1227e37c24c42ad66894146c76206 2024-11-19T15:16:12,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7ab59a6af08d4309b95c27ceba8ad23f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7ab59a6af08d4309b95c27ceba8ad23f 2024-11-19T15:16:12,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/496c54d041314f56a776ec0958dad3cc to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/496c54d041314f56a776ec0958dad3cc 2024-11-19T15:16:12,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/2c2f467d65b340319426ce7e1caa9e44 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/2c2f467d65b340319426ce7e1caa9e44 2024-11-19T15:16:12,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7db462762c484f038f37273a38dfdc78 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/7db462762c484f038f37273a38dfdc78 2024-11-19T15:16:12,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/d92d8d6c9aa14ff7835d6cb78dc154d3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/d92d8d6c9aa14ff7835d6cb78dc154d3 2024-11-19T15:16:12,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/8353fb3884ce424986b5f94620f1e8ac to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/8353fb3884ce424986b5f94620f1e8ac 2024-11-19T15:16:12,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/767fd44f579b46c583d56ff00ea27055 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/767fd44f579b46c583d56ff00ea27055 2024-11-19T15:16:12,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/15c335bccb914019bfe6ea5198253e55 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/15c335bccb914019bfe6ea5198253e55 2024-11-19T15:16:12,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/d0c3ef920661408db225f49ab50b32dc to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/d0c3ef920661408db225f49ab50b32dc 2024-11-19T15:16:12,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/151a5ad0ebf4488aaafbed5b4872dc01 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/151a5ad0ebf4488aaafbed5b4872dc01 2024-11-19T15:16:12,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/d94c0ea785b849438e862e1c8ebcbeec, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/8e68fb017a334999bb8ab2fbfcb340fe, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/78ee9404fdf14810a081d277bdfd988d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/9a3555b885ae4f029c50c8ab2dadc0ed, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c8289d55b95d4263996774d12b9792a5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c54e910543284ce59d45e7c86eebcc48, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/cf17485b961242139f181c6bcfe5283f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/48d3fad64cf442fd8f106b5e7bbe6a0b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/db2445bb7a934fd7845c6d974aaeef41, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c0204e182ac34c7fb8baad81ac20565b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/de9222a9c3eb4e71864e95e07d78d44d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/2eb54edd21bd410d8fc37a0eda036f0e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/d6a0b13240e74888a63e618fdd481198, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/ebfb097c07574a47a8237526fa7c5df7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/e7d33660b3a840619721341d31d2f53e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/26a06ca09b6548699aca0a539838b468, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/2296b12dbfc04e3bb52b91999c7d928c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/a713a652aa61461a86b09e996b2def01, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/e551e7e849034c62b8958d0789163066, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/6bc134eeb73348edac4a1748d746d0ee, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/78ecee4865b14ce49bc58eb461162e20, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/1308eed68faf4a7e90ab22943a779769, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/9683070b55634c48b8ca39b8dfc1350d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/5c0ca47b5be94f8fb6ae108abce55fe3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/f3e96979c1424d34b17887d86514d66a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/f1d25d002bf64f10a7c9ac630c72c62f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/285ed1217c304d0991f51627d8d45c97, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/b63fb89babc542038747285a544d711f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/680f7d7f88bc4c4fbd3fc38f7bfe27b1] to archive 2024-11-19T15:16:12,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:16:12,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/d94c0ea785b849438e862e1c8ebcbeec to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/d94c0ea785b849438e862e1c8ebcbeec 2024-11-19T15:16:12,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/8e68fb017a334999bb8ab2fbfcb340fe to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/8e68fb017a334999bb8ab2fbfcb340fe 2024-11-19T15:16:12,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/78ee9404fdf14810a081d277bdfd988d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/78ee9404fdf14810a081d277bdfd988d 2024-11-19T15:16:12,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/9a3555b885ae4f029c50c8ab2dadc0ed to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/9a3555b885ae4f029c50c8ab2dadc0ed 2024-11-19T15:16:12,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c8289d55b95d4263996774d12b9792a5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c8289d55b95d4263996774d12b9792a5 2024-11-19T15:16:12,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c54e910543284ce59d45e7c86eebcc48 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c54e910543284ce59d45e7c86eebcc48 2024-11-19T15:16:12,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/cf17485b961242139f181c6bcfe5283f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/cf17485b961242139f181c6bcfe5283f 2024-11-19T15:16:12,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/48d3fad64cf442fd8f106b5e7bbe6a0b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/48d3fad64cf442fd8f106b5e7bbe6a0b 2024-11-19T15:16:12,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/db2445bb7a934fd7845c6d974aaeef41 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/db2445bb7a934fd7845c6d974aaeef41 2024-11-19T15:16:12,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c0204e182ac34c7fb8baad81ac20565b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/c0204e182ac34c7fb8baad81ac20565b 2024-11-19T15:16:12,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/de9222a9c3eb4e71864e95e07d78d44d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/de9222a9c3eb4e71864e95e07d78d44d 2024-11-19T15:16:12,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/2eb54edd21bd410d8fc37a0eda036f0e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/2eb54edd21bd410d8fc37a0eda036f0e 2024-11-19T15:16:12,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/d6a0b13240e74888a63e618fdd481198 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/d6a0b13240e74888a63e618fdd481198 2024-11-19T15:16:12,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/ebfb097c07574a47a8237526fa7c5df7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/ebfb097c07574a47a8237526fa7c5df7 2024-11-19T15:16:12,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/e7d33660b3a840619721341d31d2f53e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/e7d33660b3a840619721341d31d2f53e 2024-11-19T15:16:12,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/26a06ca09b6548699aca0a539838b468 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/26a06ca09b6548699aca0a539838b468 2024-11-19T15:16:12,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/2296b12dbfc04e3bb52b91999c7d928c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/2296b12dbfc04e3bb52b91999c7d928c 2024-11-19T15:16:12,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/a713a652aa61461a86b09e996b2def01 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/a713a652aa61461a86b09e996b2def01 2024-11-19T15:16:12,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/e551e7e849034c62b8958d0789163066 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/e551e7e849034c62b8958d0789163066 2024-11-19T15:16:12,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/6bc134eeb73348edac4a1748d746d0ee to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/6bc134eeb73348edac4a1748d746d0ee 2024-11-19T15:16:12,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/78ecee4865b14ce49bc58eb461162e20 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/78ecee4865b14ce49bc58eb461162e20 2024-11-19T15:16:12,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/1308eed68faf4a7e90ab22943a779769 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/1308eed68faf4a7e90ab22943a779769 2024-11-19T15:16:12,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/9683070b55634c48b8ca39b8dfc1350d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/9683070b55634c48b8ca39b8dfc1350d 2024-11-19T15:16:12,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/5c0ca47b5be94f8fb6ae108abce55fe3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/5c0ca47b5be94f8fb6ae108abce55fe3 2024-11-19T15:16:12,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/f3e96979c1424d34b17887d86514d66a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/f3e96979c1424d34b17887d86514d66a 2024-11-19T15:16:12,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/f1d25d002bf64f10a7c9ac630c72c62f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/f1d25d002bf64f10a7c9ac630c72c62f 2024-11-19T15:16:12,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/285ed1217c304d0991f51627d8d45c97 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/285ed1217c304d0991f51627d8d45c97 2024-11-19T15:16:12,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/b63fb89babc542038747285a544d711f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/b63fb89babc542038747285a544d711f 2024-11-19T15:16:12,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/680f7d7f88bc4c4fbd3fc38f7bfe27b1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/680f7d7f88bc4c4fbd3fc38f7bfe27b1 2024-11-19T15:16:12,331 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/recovered.edits/435.seqid, newMaxSeqId=435, maxSeqId=1 2024-11-19T15:16:12,332 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. 2024-11-19T15:16:12,332 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 9d89d22f9c07cea236aa6c26f89b6b4f: Waiting for close lock at 1732029371416Running coprocessor pre-close hooks at 1732029371416Disabling compacts and flushes for region at 1732029371416Disabling writes for close at 1732029371416Obtaining lock to block concurrent updates at 1732029371416Preparing flush snapshotting stores in 9d89d22f9c07cea236aa6c26f89b6b4f at 1732029371416Finished memstore snapshotting TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732029371417 (+1 ms)Flushing stores of TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f. at 1732029371417Flushing 9d89d22f9c07cea236aa6c26f89b6b4f/A: creating writer at 1732029371417Flushing 9d89d22f9c07cea236aa6c26f89b6b4f/A: appending metadata at 1732029371419 (+2 ms)Flushing 9d89d22f9c07cea236aa6c26f89b6b4f/A: closing flushed file at 1732029371419Flushing 9d89d22f9c07cea236aa6c26f89b6b4f/B: creating writer at 1732029371427 (+8 ms)Flushing 9d89d22f9c07cea236aa6c26f89b6b4f/B: appending metadata at 1732029371429 (+2 ms)Flushing 9d89d22f9c07cea236aa6c26f89b6b4f/B: closing flushed file at 1732029371429Flushing 9d89d22f9c07cea236aa6c26f89b6b4f/C: creating writer at 1732029371836 (+407 ms)Flushing 9d89d22f9c07cea236aa6c26f89b6b4f/C: appending metadata at 1732029371838 (+2 ms)Flushing 9d89d22f9c07cea236aa6c26f89b6b4f/C: closing flushed file at 1732029371838Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@734896a: reopening flushed file at 1732029372244 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1989d706: reopening flushed file at 1732029372248 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6500097d: reopening flushed file at 1732029372252 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9d89d22f9c07cea236aa6c26f89b6b4f in 839ms, sequenceid=432, compaction requested=true at 1732029372256 (+4 ms)Writing region close event to WAL at 1732029372329 (+73 ms)Running coprocessor post-close hooks at 1732029372332 (+3 ms)Closed at 1732029372332 2024-11-19T15:16:12,334 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=9d89d22f9c07cea236aa6c26f89b6b4f, regionState=CLOSED 2024-11-19T15:16:12,334 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:16:12,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:16:12,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-19T15:16:12,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 9d89d22f9c07cea236aa6c26f89b6b4f, server=50570b5b74e7,46557,1732029218639 in 1.0720 sec 2024-11-19T15:16:12,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-19T15:16:12,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d89d22f9c07cea236aa6c26f89b6b4f, UNASSIGN in 1.0750 sec 2024-11-19T15:16:12,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-19T15:16:12,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0780 sec 2024-11-19T15:16:12,341 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029372341"}]},"ts":"1732029372341"} 2024-11-19T15:16:12,342 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T15:16:12,342 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T15:16:12,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0860 sec 2024-11-19T15:16:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T15:16:12,394 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:12,395 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T15:16:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:12,396 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T15:16:12,396 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:12,398 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:16:12,400 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/recovered.edits] 2024-11-19T15:16:12,401 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/238502d2a75744f4a2f9e5b2d9729652 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/238502d2a75744f4a2f9e5b2d9729652 2024-11-19T15:16:12,402 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/66ef0e1f5c3d4ca89837034a01b00045 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/66ef0e1f5c3d4ca89837034a01b00045 2024-11-19T15:16:12,403 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/b71dff15354040bc85b5899c9c234251 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/A/b71dff15354040bc85b5899c9c234251 2024-11-19T15:16:12,405 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/026f5d05dc064c87a8a6c13e4536e3e9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/026f5d05dc064c87a8a6c13e4536e3e9 2024-11-19T15:16:12,406 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/1180f6b72a00444c9ec073d6277809c3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/1180f6b72a00444c9ec073d6277809c3 2024-11-19T15:16:12,406 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/11a1978f07ba46899f26f6a9e060e76e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/B/11a1978f07ba46899f26f6a9e060e76e 2024-11-19T15:16:12,408 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/4072645226ad44ce94277fe69aab2681 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/4072645226ad44ce94277fe69aab2681 2024-11-19T15:16:12,409 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/4d7ed1ff09b343d9852358b837bf3190 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/4d7ed1ff09b343d9852358b837bf3190 2024-11-19T15:16:12,410 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/ab927eed683045078437b6cc86c2dd5d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/C/ab927eed683045078437b6cc86c2dd5d 2024-11-19T15:16:12,412 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/recovered.edits/435.seqid to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f/recovered.edits/435.seqid 2024-11-19T15:16:12,412 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/9d89d22f9c07cea236aa6c26f89b6b4f 2024-11-19T15:16:12,412 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T15:16:12,414 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:12,416 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T15:16:12,417 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T15:16:12,418 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:12,418 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T15:16:12,418 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732029372418"}]},"ts":"9223372036854775807"} 2024-11-19T15:16:12,420 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T15:16:12,420 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9d89d22f9c07cea236aa6c26f89b6b4f, NAME => 'TestAcidGuarantees,,1732029345333.9d89d22f9c07cea236aa6c26f89b6b4f.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T15:16:12,420 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T15:16:12,420 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732029372420"}]},"ts":"9223372036854775807"} 2024-11-19T15:16:12,421 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T15:16:12,422 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:12,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-11-19T15:16:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T15:16:12,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T15:16:12,504 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:12,513 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 250), OpenFileDescriptor=447 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=439 (was 418) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7328 (was 7380) 2024-11-19T15:16:12,522 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=439, ProcessCount=11, AvailableMemoryMB=7328 2024-11-19T15:16:12,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T15:16:12,523 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T15:16:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:12,524 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T15:16:12,525 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:12,525 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-19T15:16:12,525 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T15:16:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T15:16:12,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742369_1545 (size=1061) 2024-11-19T15:16:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T15:16:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T15:16:12,931 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8 2024-11-19T15:16:12,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742370_1546 (size=53) 2024-11-19T15:16:13,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T15:16:13,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:16:13,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 017a8429196619e4c293e69ec8b529c1, disabling compactions & flushes 2024-11-19T15:16:13,336 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:13,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:13,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. after waiting 0 ms 2024-11-19T15:16:13,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:13,336 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:13,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 017a8429196619e4c293e69ec8b529c1: Waiting for close lock at 1732029373336Disabling compacts and flushes for region at 1732029373336Disabling writes for close at 1732029373336Writing region close event to WAL at 1732029373336Closed at 1732029373336 2024-11-19T15:16:13,337 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T15:16:13,337 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732029373337"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732029373337"}]},"ts":"1732029373337"} 2024-11-19T15:16:13,339 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T15:16:13,339 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T15:16:13,339 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029373339"}]},"ts":"1732029373339"} 2024-11-19T15:16:13,340 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T15:16:13,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, ASSIGN}] 2024-11-19T15:16:13,341 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, ASSIGN 2024-11-19T15:16:13,342 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, ASSIGN; state=OFFLINE, location=50570b5b74e7,46557,1732029218639; forceNewPlan=false, retain=false 2024-11-19T15:16:13,492 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=017a8429196619e4c293e69ec8b529c1, regionState=OPENING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:13,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, ASSIGN because future has completed 2024-11-19T15:16:13,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:16:13,648 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:13,648 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T15:16:13,649 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,649 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:16:13,649 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,649 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,650 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,651 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:16:13,651 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 017a8429196619e4c293e69ec8b529c1 columnFamilyName A 2024-11-19T15:16:13,651 DEBUG [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:13,651 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.HStore(327): Store=017a8429196619e4c293e69ec8b529c1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:16:13,651 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,653 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:16:13,653 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 017a8429196619e4c293e69ec8b529c1 columnFamilyName B 2024-11-19T15:16:13,653 DEBUG [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:13,654 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.HStore(327): Store=017a8429196619e4c293e69ec8b529c1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:16:13,654 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,655 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:16:13,655 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 017a8429196619e4c293e69ec8b529c1 columnFamilyName C 2024-11-19T15:16:13,655 DEBUG [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:13,655 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.HStore(327): Store=017a8429196619e4c293e69ec8b529c1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:16:13,656 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:13,656 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,656 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,656 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,657 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,657 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,658 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T15:16:13,659 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,660 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T15:16:13,660 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 017a8429196619e4c293e69ec8b529c1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70467268, jitterRate=0.050044119358062744}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T15:16:13,661 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:13,661 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 017a8429196619e4c293e69ec8b529c1: Running coprocessor pre-open hook at 1732029373649Writing region info on filesystem at 1732029373649Initializing all the Stores at 1732029373649Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029373649Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029373650 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029373650Cleaning up temporary data from old regions at 1732029373657 (+7 ms)Running coprocessor post-open hooks at 1732029373661 (+4 ms)Region opened successfully at 1732029373661 2024-11-19T15:16:13,662 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., pid=148, masterSystemTime=1732029373646 2024-11-19T15:16:13,663 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:13,663 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T15:16:13,664 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=017a8429196619e4c293e69ec8b529c1, regionState=OPEN, openSeqNum=2, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:13,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:16:13,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-19T15:16:13,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 in 172 msec 2024-11-19T15:16:13,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-19T15:16:13,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, ASSIGN in 326 msec 2024-11-19T15:16:13,670 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T15:16:13,670 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029373670"}]},"ts":"1732029373670"} 2024-11-19T15:16:13,671 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T15:16:13,672 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T15:16:13,673 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-19T15:16:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T15:16:14,674 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:14,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5581e3e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:14,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:16:14,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:16:14,676 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:16:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:16:14,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:16:14,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3769dd7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:14,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:16:14,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:16:14,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:14,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:16:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0ab0b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:14,678 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:16:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:16:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:16:14,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:16:14,680 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T15:16:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 50570b5b74e7,39501,1732029217890 2024-11-19T15:16:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@59451b9f 2024-11-19T15:16:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T15:16:14,682 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T15:16:14,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T15:16:14,683 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T15:16:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:14,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742371_1547 (size=1097) 2024-11-19T15:16:15,093 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T15:16:15,093 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T15:16:15,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T15:16:15,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, REOPEN/MOVE}] 2024-11-19T15:16:15,098 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, REOPEN/MOVE 2024-11-19T15:16:15,098 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=017a8429196619e4c293e69ec8b529c1, regionState=CLOSING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, REOPEN/MOVE because future has completed 2024-11-19T15:16:15,099 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T15:16:15,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:16:15,251 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,251 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T15:16:15,251 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 017a8429196619e4c293e69ec8b529c1, disabling compactions & flushes 2024-11-19T15:16:15,252 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:15,252 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:15,252 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. after waiting 0 ms 2024-11-19T15:16:15,252 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:15,255 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T15:16:15,255 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:15,255 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 017a8429196619e4c293e69ec8b529c1: Waiting for close lock at 1732029375251Running coprocessor pre-close hooks at 1732029375251Disabling compacts and flushes for region at 1732029375251Disabling writes for close at 1732029375252 (+1 ms)Writing region close event to WAL at 1732029375252Running coprocessor post-close hooks at 1732029375255 (+3 ms)Closed at 1732029375255 2024-11-19T15:16:15,255 WARN [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 017a8429196619e4c293e69ec8b529c1 to self. 2024-11-19T15:16:15,257 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,257 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=017a8429196619e4c293e69ec8b529c1, regionState=CLOSED 2024-11-19T15:16:15,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:16:15,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-19T15:16:15,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 in 160 msec 2024-11-19T15:16:15,261 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, REOPEN/MOVE; state=CLOSED, location=50570b5b74e7,46557,1732029218639; forceNewPlan=false, retain=true 2024-11-19T15:16:15,411 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=017a8429196619e4c293e69ec8b529c1, regionState=OPENING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, REOPEN/MOVE because future has completed 2024-11-19T15:16:15,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:16:15,567 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:15,567 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T15:16:15,567 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,567 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T15:16:15,568 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,568 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,569 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,569 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:16:15,569 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 017a8429196619e4c293e69ec8b529c1 columnFamilyName A 2024-11-19T15:16:15,570 DEBUG [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:15,571 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.HStore(327): Store=017a8429196619e4c293e69ec8b529c1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:16:15,571 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,571 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:16:15,572 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 017a8429196619e4c293e69ec8b529c1 columnFamilyName B 2024-11-19T15:16:15,572 DEBUG [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:15,572 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.HStore(327): Store=017a8429196619e4c293e69ec8b529c1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:16:15,572 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,573 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T15:16:15,573 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 017a8429196619e4c293e69ec8b529c1 columnFamilyName C 2024-11-19T15:16:15,573 DEBUG [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:15,573 INFO [StoreOpener-017a8429196619e4c293e69ec8b529c1-1 {}] regionserver.HStore(327): Store=017a8429196619e4c293e69ec8b529c1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T15:16:15,573 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:15,573 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,574 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,574 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,575 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,575 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,576 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T15:16:15,576 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,577 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 017a8429196619e4c293e69ec8b529c1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61158451, jitterRate=-0.08866806328296661}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T15:16:15,577 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,578 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 017a8429196619e4c293e69ec8b529c1: Running coprocessor pre-open hook at 1732029375568Writing region info on filesystem at 1732029375568Initializing all the Stores at 1732029375568Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029375568Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029375569 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732029375569Cleaning up temporary data from old regions at 1732029375575 (+6 ms)Running coprocessor post-open hooks at 1732029375577 (+2 ms)Region opened successfully at 1732029375578 (+1 ms) 2024-11-19T15:16:15,578 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., pid=153, masterSystemTime=1732029375564 2024-11-19T15:16:15,580 DEBUG [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:15,580 INFO [RS_OPEN_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:15,581 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=017a8429196619e4c293e69ec8b529c1, regionState=OPEN, openSeqNum=5, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:16:15,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-19T15:16:15,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 in 170 msec 2024-11-19T15:16:15,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-19T15:16:15,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, REOPEN/MOVE in 487 msec 2024-11-19T15:16:15,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-19T15:16:15,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-19T15:16:15,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 905 msec 2024-11-19T15:16:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-19T15:16:15,590 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:15,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,590 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:16:15,590 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:16:15,591 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:16:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:16:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:16:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14af61f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:16:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:16:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:15,593 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:16:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@591e3c5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:16:15,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:16:15,594 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:16:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:16:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:16:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55875e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:16:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:16:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:15,596 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:16:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1a7ac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:16:15,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:16:15,597 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:16:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:16:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:16:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f786092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:16:15,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:16:15,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:15,598 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:16:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd27f9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,599 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:16:15,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:16:15,600 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:16:15,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:16:15,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:16:15,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b3fef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:16:15,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:16:15,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:15,601 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:16:15,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a809941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:16:15,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:16:15,602 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:16:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:16:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:16:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fdf3991, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:16:15,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:16:15,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:15,603 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:16:15,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75403f10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:16:15,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:16:15,604 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:16:15,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:16:15,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:16:15,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31113a15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:16:15,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:16:15,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:15,606 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:16:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3243bb5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:16:15,607 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:16:15,607 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:16:15,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:16:15,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:16:15,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eba1f48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:16:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:16:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:15,608 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:16:15,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30337caf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:16:15,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:16:15,610 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:16:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:16:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:16:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c23742a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:16:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:16:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:15,611 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:16:15,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d2f1628, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:16:15,612 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:16:15,613 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:16:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:16:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:16:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@624425f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:16:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:16:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:15,614 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:16:15,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b7ea06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8fc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 50570b5b74e7,39501,-1 for getting cluster id 2024-11-19T15:16:15,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T15:16:15,615 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3dd3724b-b902-4c12-9084-cb3290376b64' 2024-11-19T15:16:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T15:16:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3dd3724b-b902-4c12-9084-cb3290376b64" 2024-11-19T15:16:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7104bf64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [50570b5b74e7,39501,-1] 2024-11-19T15:16:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T15:16:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:15,616 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T15:16:15,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@791529a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T15:16:15,617 DEBUG [Thread-2444 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:16:15,618 DEBUG [Thread-2455 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:16:15,618 DEBUG [Thread-2442 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:16:15,618 DEBUG [Thread-2446 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:16:15,618 DEBUG [Thread-2438 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:16:15,618 DEBUG [Thread-2451 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:16:15,618 DEBUG [Thread-2457 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:16:15,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:16:15,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:16:15,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:16:15,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:16:15,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:16:15,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:16:15,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:16:15,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:16:15,619 DEBUG [Thread-2440 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:16:15,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:16:15,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:16:15,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:16:15,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:16:15,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:16:15,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:16:15,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:16:15,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:16:15,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:16:15,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:16:15,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:16:15,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:16:15,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:16:15,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:16:15,621 DEBUG [Thread-2449 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:16:15,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:16:15,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:16:15,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:16:15,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:16:15,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:16:15,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:16:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:16:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:16:15,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:16:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:16:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:16:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:16:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:16:15,625 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:16:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:16:15,626 DEBUG [Thread-2453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T15:16:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-19T15:16:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=50570b5b74e7,46557,1732029218639, seqNum=-1] 2024-11-19T15:16:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T15:16:15,627 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:16:15,627 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T15:16:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T15:16:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5] 2024-11-19T15:16:15,629 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:16:15,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:16:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:15,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:16:15,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:15,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:15,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:15,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:15,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:15,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:15,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732029375649, max timestamp=1732029375655Num uniques 859; 2024-11-19T15:16:15,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T15:16:15,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029375649, max timestamp=1732029375655Num uniques 859; 2024-11-19T15:16:15,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:16:15,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029375649, max timestamp=1732029375655Num uniques 859; 2024-11-19T15:16:15,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:16:15,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196ee00c7b26284e849a675616216414f2_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029375628/Put/seqid=0 2024-11-19T15:16:15,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742372_1548 (size=14594) 2024-11-19T15:16:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029435678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:15,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029435681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:15,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029435681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029435682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029435684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T15:16:15,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T15:16:15,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:15,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:15,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:15,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:15,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:15,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029435794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029435794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029435794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029435795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:15,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029435795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:15,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T15:16:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:15,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:15,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:15,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:15,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T15:16:16,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029436004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029436004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029436004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029436004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029436004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,076 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,079 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196ee00c7b26284e849a675616216414f2_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196ee00c7b26284e849a675616216414f2_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:16,080 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/2a619cf2ff0f4311aea60a19aaca3faf, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:16,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/2a619cf2ff0f4311aea60a19aaca3faf is 175, key is test_row_0/A:col10/1732029375628/Put/seqid=0 2024-11-19T15:16:16,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742373_1549 (size=39549) 2024-11-19T15:16:16,084 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/2a619cf2ff0f4311aea60a19aaca3faf 2024-11-19T15:16:16,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T15:16:16,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:16,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:16,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:16,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:16,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:16,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/9f7f9e1cc6084eb2898999ccbec86467 is 50, key is test_row_0/B:col10/1732029375628/Put/seqid=0 2024-11-19T15:16:16,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742374_1550 (size=12001) 2024-11-19T15:16:16,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/9f7f9e1cc6084eb2898999ccbec86467 2024-11-19T15:16:16,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/7de4254f979c416f80b36e8f230eb064 is 50, key is test_row_0/C:col10/1732029375628/Put/seqid=0 2024-11-19T15:16:16,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742375_1551 (size=12001) 2024-11-19T15:16:16,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/7de4254f979c416f80b36e8f230eb064 2024-11-19T15:16:16,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/2a619cf2ff0f4311aea60a19aaca3faf as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/2a619cf2ff0f4311aea60a19aaca3faf 2024-11-19T15:16:16,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/2a619cf2ff0f4311aea60a19aaca3faf, entries=200, sequenceid=15, filesize=38.6 K 2024-11-19T15:16:16,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/9f7f9e1cc6084eb2898999ccbec86467 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/9f7f9e1cc6084eb2898999ccbec86467 2024-11-19T15:16:16,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/9f7f9e1cc6084eb2898999ccbec86467, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T15:16:16,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/7de4254f979c416f80b36e8f230eb064 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7de4254f979c416f80b36e8f230eb064 2024-11-19T15:16:16,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7de4254f979c416f80b36e8f230eb064, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T15:16:16,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for 017a8429196619e4c293e69ec8b529c1 in 533ms, sequenceid=15, compaction requested=false 2024-11-19T15:16:16,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:16,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T15:16:16,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:16,245 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-11-19T15:16:16,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:16,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:16,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:16,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:16,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:16,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ac56bcf6ba824f70a1d1897b474f3bbf_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029375676/Put/seqid=0 2024-11-19T15:16:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T15:16:16,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742376_1552 (size=12154) 2024-11-19T15:16:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:16,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:16,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029436320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029436321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029436321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029436325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029436326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029436434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029436434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029436434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029436434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029436444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029436644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029436644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029436644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029436644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029436654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:16,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,670 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ac56bcf6ba824f70a1d1897b474f3bbf_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac56bcf6ba824f70a1d1897b474f3bbf_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:16,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/624679725a3e4509aaabb49d4fae5136, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:16,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/624679725a3e4509aaabb49d4fae5136 is 175, key is test_row_0/A:col10/1732029375676/Put/seqid=0 2024-11-19T15:16:16,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742377_1553 (size=30955) 2024-11-19T15:16:16,675 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/624679725a3e4509aaabb49d4fae5136 2024-11-19T15:16:16,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/1a5f68f1d5564eee9eb476f888df14b0 is 50, key is test_row_0/B:col10/1732029375676/Put/seqid=0 2024-11-19T15:16:16,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742378_1554 (size=12001) 2024-11-19T15:16:16,684 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/1a5f68f1d5564eee9eb476f888df14b0 2024-11-19T15:16:16,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/4fcfe0996bbd48fc98827e038c1d7da0 is 50, key is test_row_0/C:col10/1732029375676/Put/seqid=0 2024-11-19T15:16:16,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742379_1555 (size=12001) 2024-11-19T15:16:16,702 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/4fcfe0996bbd48fc98827e038c1d7da0 2024-11-19T15:16:16,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/624679725a3e4509aaabb49d4fae5136 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/624679725a3e4509aaabb49d4fae5136 2024-11-19T15:16:16,710 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/624679725a3e4509aaabb49d4fae5136, entries=150, sequenceid=51, filesize=30.2 K 2024-11-19T15:16:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/1a5f68f1d5564eee9eb476f888df14b0 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/1a5f68f1d5564eee9eb476f888df14b0 2024-11-19T15:16:16,714 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/1a5f68f1d5564eee9eb476f888df14b0, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T15:16:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/4fcfe0996bbd48fc98827e038c1d7da0 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/4fcfe0996bbd48fc98827e038c1d7da0 2024-11-19T15:16:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,720 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/4fcfe0996bbd48fc98827e038c1d7da0, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T15:16:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,720 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for 017a8429196619e4c293e69ec8b529c1 in 475ms, sequenceid=51, compaction requested=false 2024-11-19T15:16:16,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:16,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:16,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-19T15:16:16,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-19T15:16:16,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-19T15:16:16,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0930 sec 2024-11-19T15:16:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.0990 sec 2024-11-19T15:16:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T15:16:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,766 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:16,766 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:16:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-19T15:16:16,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T15:16:16,768 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:16:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,769 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:16:16,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:16:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T15:16:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T15:16:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,923 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T15:16:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:16,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198f8aced484a04e6d96a038f9d204325d_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029376325/Put/seqid=0 2024-11-19T15:16:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742380_1556 (size=9714) 2024-11-19T15:16:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:16,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:17,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029437003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029437004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029376993, max timestamp=1732029377007Num uniques 859; 2024-11-19T15:16:17,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:16:17,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029376993, max timestamp=1732029377007Num uniques 859; 2024-11-19T15:16:17,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:16:17,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732029376993, max timestamp=1732029377007Num uniques 859; 2024-11-19T15:16:17,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T15:16:17,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029437038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029437039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029437042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T15:16:17,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029437114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029437114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029437154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029437155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029437155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029437324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029437324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:17,336 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198f8aced484a04e6d96a038f9d204325d_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198f8aced484a04e6d96a038f9d204325d_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/f66c8f995eb2463f9e245f032ee63994, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:17,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/f66c8f995eb2463f9e245f032ee63994 is 175, key is test_row_0/A:col10/1732029376325/Put/seqid=0 2024-11-19T15:16:17,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742381_1557 (size=22361) 2024-11-19T15:16:17,343 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/f66c8f995eb2463f9e245f032ee63994 2024-11-19T15:16:17,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/feacff36651d4c9c9c9d82e98d23f737 is 50, key is test_row_0/B:col10/1732029376325/Put/seqid=0 2024-11-19T15:16:17,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742382_1558 (size=9657) 2024-11-19T15:16:17,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029437364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029437364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029437364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T15:16:17,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029437634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029437634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029437674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029437675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029437675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:17,753 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/feacff36651d4c9c9c9d82e98d23f737 2024-11-19T15:16:17,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/10ad70c2b8e24234b99f9ce70968b883 is 50, key is test_row_0/C:col10/1732029376325/Put/seqid=0 2024-11-19T15:16:17,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742383_1559 (size=9657) 2024-11-19T15:16:17,853 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T15:16:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T15:16:18,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029438144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:18,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029438144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:18,164 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/10ad70c2b8e24234b99f9ce70968b883 2024-11-19T15:16:18,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/f66c8f995eb2463f9e245f032ee63994 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f66c8f995eb2463f9e245f032ee63994 2024-11-19T15:16:18,171 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f66c8f995eb2463f9e245f032ee63994, entries=100, sequenceid=61, filesize=21.8 K 2024-11-19T15:16:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/feacff36651d4c9c9c9d82e98d23f737 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/feacff36651d4c9c9c9d82e98d23f737 2024-11-19T15:16:18,175 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/feacff36651d4c9c9c9d82e98d23f737, entries=100, sequenceid=61, filesize=9.4 K 2024-11-19T15:16:18,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/10ad70c2b8e24234b99f9ce70968b883 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/10ad70c2b8e24234b99f9ce70968b883 2024-11-19T15:16:18,178 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/10ad70c2b8e24234b99f9ce70968b883, entries=100, sequenceid=61, filesize=9.4 K 2024-11-19T15:16:18,179 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for 017a8429196619e4c293e69ec8b529c1 in 1256ms, sequenceid=61, compaction requested=true 2024-11-19T15:16:18,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-19T15:16:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-19T15:16:18,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-19T15:16:18,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-19T15:16:18,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4170 sec 2024-11-19T15:16:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:18,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=140.49 KB heapSize=404.70 KB 2024-11-19T15:16:18,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732029377031, max timestamp=1732029378187Num uniques 1018; 2024-11-19T15:16:18,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T15:16:18,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732029377031, max timestamp=1732029378187Num uniques 1018; 2024-11-19T15:16:18,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T15:16:18,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732029377031, max timestamp=1732029378187Num uniques 1009; 2024-11-19T15:16:18,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-19T15:16:18,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:18,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ba404de488f5407bb515a4c4de8d811a_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029377038/Put/seqid=0 2024-11-19T15:16:18,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742384_1560 (size=12154) 2024-11-19T15:16:18,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732029378210, max timestamp=1732029378225Num uniques 859; 2024-11-19T15:16:18,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T15:16:18,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029378215, max timestamp=1732029378225Num uniques 859; 2024-11-19T15:16:18,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:16:18,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732029378215, max timestamp=1732029378225Num uniques 859; 2024-11-19T15:16:18,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T15:16:18,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732029378258, max timestamp=1732029378265Num uniques 1018; 2024-11-19T15:16:18,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732029378258, max timestamp=1732029378265Num uniques 1018; 2024-11-19T15:16:18,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732029378258, max timestamp=1732029378265Num uniques 1068; 2024-11-19T15:16:18,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-19T15:16:18,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T15:16:18,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T15:16:18,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732029378295, max timestamp=1732029378303Num uniques 1027; 2024-11-19T15:16:18,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T15:16:18,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732029378295, max timestamp=1732029378303Num uniques 1027; 2024-11-19T15:16:18,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T15:16:18,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732029378295, max timestamp=1732029378303Num uniques 1027; 2024-11-19T15:16:18,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648  suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T15:16:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732029378312, max timestamp=1732029378345Num uniques 1036; 2024-11-19T15:16:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732029378312, max timestamp=1732029378345Num uniques 1036; 2024-11-19T15:16:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-19T15:16:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732029378312, max timestamp=1732029378345Num uniques 1036; 2024-11-19T15:16:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-19T15:16:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784  suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-19T15:16:18,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732029378365, max timestamp=1732029378386Num uniques 1095; 2024-11-19T15:16:18,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T15:16:18,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732029378365, max timestamp=1732029378386Num uniques 1095; 2024-11-19T15:16:18,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T15:16:18,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732029378365, max timestamp=1732029378386Num uniques 1095; 2024-11-19T15:16:18,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720  suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T15:16:18,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029438454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:18,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732029378444, max timestamp=1732029378458Num uniques 1104; 2024-11-19T15:16:18,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-19T15:16:18,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732029378444, max timestamp=1732029378458Num uniques 1104; 2024-11-19T15:16:18,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732029378444, max timestamp=1732029378458Num uniques 1104; 2024-11-19T15:16:18,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-19T15:16:18,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-19T15:16:18,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732029378482, max timestamp=1732029378516Num uniques 1063; 2024-11-19T15:16:18,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-19T15:16:18,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732029378482, max timestamp=1732029378516Num uniques 1063; 2024-11-19T15:16:18,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-19T15:16:18,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732029378482, max timestamp=1732029378516Num uniques 1063; 2024-11-19T15:16:18,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-19T15:16:18,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732029378543, max timestamp=1732029378566Num uniques 1022; 2024-11-19T15:16:18,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328  suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-19T15:16:18,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732029378543, max timestamp=1732029378566Num uniques 1022; 2024-11-19T15:16:18,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328  suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-19T15:16:18,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732029378543, max timestamp=1732029378566Num uniques 1022; 2024-11-19T15:16:18,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328  suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-19T15:16:18,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732029378594, max timestamp=1732029378608Num uniques 1081; 2024-11-19T15:16:18,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732029378594, max timestamp=1732029378608Num uniques 1081; 2024-11-19T15:16:18,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264  suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-19T15:16:18,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264  suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-19T15:16:18,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732029378594, max timestamp=1732029378608Num uniques 1081; 2024-11-19T15:16:18,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264  suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-19T15:16:18,614 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:18,617 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ba404de488f5407bb515a4c4de8d811a_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ba404de488f5407bb515a4c4de8d811a_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:18,618 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/db6ac21e756644fbacbfbc373a9c8015, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:18,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/db6ac21e756644fbacbfbc373a9c8015 is 175, key is test_row_0/A:col10/1732029377038/Put/seqid=0 2024-11-19T15:16:18,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742385_1561 (size=30955) 2024-11-19T15:16:18,628 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/db6ac21e756644fbacbfbc373a9c8015 2024-11-19T15:16:18,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/dd75489a178f4dd3b4eb0de69e86d23a is 50, key is test_row_0/B:col10/1732029377038/Put/seqid=0 2024-11-19T15:16:18,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742386_1562 (size=12001) 2024-11-19T15:16:18,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732029378641, max timestamp=1732029378655Num uniques 1090; 2024-11-19T15:16:18,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200  suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-19T15:16:18,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732029378641, max timestamp=1732029378655Num uniques 1090; 2024-11-19T15:16:18,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732029378641, max timestamp=1732029378655Num uniques 1090; 2024-11-19T15:16:18,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200  suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-19T15:16:18,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200  suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-19T15:16:18,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029438694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:18,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732029378682, max timestamp=1732029378696Num uniques 1099; 2024-11-19T15:16:18,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732029378682, max timestamp=1732029378696Num uniques 1099; 2024-11-19T15:16:18,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732029378682, max timestamp=1732029378696Num uniques 1099; 2024-11-19T15:16:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:18,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936  suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-19T15:16:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:18,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936  suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-19T15:16:18,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936  suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-19T15:16:18,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732029378740, max timestamp=1732029378752Num uniques 1058; 2024-11-19T15:16:18,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732029378740, max timestamp=1732029378752Num uniques 1058; 2024-11-19T15:16:18,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672  suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-19T15:16:18,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672  suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-19T15:16:18,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732029378740, max timestamp=1732029378752Num uniques 1058; 2024-11-19T15:16:18,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672  suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-19T15:16:18,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732029378787, max timestamp=1732029378803Num uniques 1017; 2024-11-19T15:16:18,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808  suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-19T15:16:18,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732029378787, max timestamp=1732029378803Num uniques 1017; 2024-11-19T15:16:18,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732029378787, max timestamp=1732029378803Num uniques 1017; 2024-11-19T15:16:18,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808  suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-19T15:16:18,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808  suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-19T15:16:18,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1732029378833, max timestamp=1732029378845Num uniques 1076; 2024-11-19T15:16:18,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=12636, suffix heap size=139872,new segment heap size=28944  suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=276 2024-11-19T15:16:18,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1732029378833, max timestamp=1732029378845Num uniques 1076; 2024-11-19T15:16:18,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=12636, suffix heap size=139872,new segment heap size=28944  suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=276 2024-11-19T15:16:18,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1732029378833, max timestamp=1732029378845Num uniques 1076; 2024-11-19T15:16:18,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=12636, suffix heap size=139872,new segment heap size=28944  suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=276 2024-11-19T15:16:18,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029438885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:18,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1732029378837, max timestamp=1732029378889Num uniques 1135; 2024-11-19T15:16:18,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=13050, suffix heap size=146008,new segment heap size=29880  suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=285 2024-11-19T15:16:18,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1732029378837, max timestamp=1732029378889Num uniques 1135; 2024-11-19T15:16:18,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=13050, suffix heap size=146008,new segment heap size=29880  suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=285 2024-11-19T15:16:18,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1732029378837, max timestamp=1732029378889Num uniques 1135; 2024-11-19T15:16:18,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=13050, suffix heap size=146008,new segment heap size=29880  suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=285 2024-11-19T15:16:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T15:16:18,915 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:18,915 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:16:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-19T15:16:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T15:16:18,916 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:16:18,917 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:16:18,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:16:18,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1732029378933, max timestamp=1732029378948Num uniques 1144; 2024-11-19T15:16:18,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1732029378933, max timestamp=1732029378948Num uniques 1144; 2024-11-19T15:16:18,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=11174, suffix heap size=146944,new segment heap size=25616  suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=244 2024-11-19T15:16:18,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=11174, suffix heap size=146944,new segment heap size=25616  suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=244 2024-11-19T15:16:18,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1732029378933, max timestamp=1732029378948Num uniques 1144; 2024-11-19T15:16:18,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=11174, suffix heap size=146944,new segment heap size=25616  suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=244 2024-11-19T15:16:19,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029439004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029439004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1732029378988, max timestamp=1732029379006Num uniques 1103; 2024-11-19T15:16:19,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1732029378988, max timestamp=1732029379006Num uniques 1103; 2024-11-19T15:16:19,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352  suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-11-19T15:16:19,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1732029378988, max timestamp=1732029379006Num uniques 1103; 2024-11-19T15:16:19,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352  suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-11-19T15:16:19,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352  suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-11-19T15:16:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T15:16:19,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/dd75489a178f4dd3b4eb0de69e86d23a 2024-11-19T15:16:19,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/17621af7af784cf8a087af60e0091eb2 is 50, key is test_row_0/C:col10/1732029377038/Put/seqid=0 2024-11-19T15:16:19,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742387_1563 (size=12001) 2024-11-19T15:16:19,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:19,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:19,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1732029379064, max timestamp=1732029379119Num uniques 1062; 2024-11-19T15:16:19,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1732029379064, max timestamp=1732029379119Num uniques 1062; 2024-11-19T15:16:19,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288  suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-11-19T15:16:19,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288  suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-11-19T15:16:19,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1732029379064, max timestamp=1732029379119Num uniques 1062; 2024-11-19T15:16:19,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288  suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-11-19T15:16:19,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=271, cellSize=12406, totalHeapSize=28424, min timestamp=1732029379153, max timestamp=1732029379168Num uniques 1071; 2024-11-19T15:16:19,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=12406, suffix heap size=139352,new segment heap size=28424  suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=271 2024-11-19T15:16:19,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=271, cellSize=12406, totalHeapSize=28424, min timestamp=1732029379153, max timestamp=1732029379168Num uniques 1071; 2024-11-19T15:16:19,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=12406, suffix heap size=139352,new segment heap size=28424  suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=271 2024-11-19T15:16:19,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=271, cellSize=12406, totalHeapSize=28424, min timestamp=1732029379153, max timestamp=1732029379168Num uniques 1071; 2024-11-19T15:16:19,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=12406, suffix heap size=139352,new segment heap size=28424  suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=271 2024-11-19T15:16:19,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029439194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029439195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732029379185, max timestamp=1732029379198Num uniques 1130; 2024-11-19T15:16:19,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51750, new segment data size=10530, suffix heap size=145488,new segment heap size=24160  suffix off heap size=0, new segment off heap size=0, suffix cells count=1130, new segment cells count=230 2024-11-19T15:16:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732029379185, max timestamp=1732029379198Num uniques 1130; 2024-11-19T15:16:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51750, new segment data size=10530, suffix heap size=145488,new segment heap size=24160  suffix off heap size=0, new segment off heap size=0, suffix cells count=1130, new segment cells count=230 2024-11-19T15:16:19,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732029379185, max timestamp=1732029379198Num uniques 1130; 2024-11-19T15:16:19,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51750, new segment data size=10530, suffix heap size=145488,new segment heap size=24160  suffix off heap size=0, new segment off heap size=0, suffix cells count=1130, new segment cells count=230 2024-11-19T15:16:19,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:19,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:19,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T15:16:19,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1732029379225, max timestamp=1732029379243Num uniques 1089; 2024-11-19T15:16:19,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=13234, suffix heap size=141224,new segment heap size=30296  suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=289 2024-11-19T15:16:19,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1732029379225, max timestamp=1732029379243Num uniques 1089; 2024-11-19T15:16:19,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=13234, suffix heap size=141224,new segment heap size=30296  suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=289 2024-11-19T15:16:19,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1732029379225, max timestamp=1732029379243Num uniques 1089; 2024-11-19T15:16:19,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=13234, suffix heap size=141224,new segment heap size=30296  suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=289 2024-11-19T15:16:19,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13648, totalHeapSize=31232, min timestamp=1732029379265, max timestamp=1732029379281Num uniques 1148; 2024-11-19T15:16:19,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13648, totalHeapSize=31232, min timestamp=1732029379265, max timestamp=1732029379281Num uniques 1148; 2024-11-19T15:16:19,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=13648, suffix heap size=147360,new segment heap size=31232  suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=298 2024-11-19T15:16:19,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=13648, suffix heap size=147360,new segment heap size=31232  suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=298 2024-11-19T15:16:19,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13648, totalHeapSize=31232, min timestamp=1732029379265, max timestamp=1732029379281Num uniques 1148; 2024-11-19T15:16:19,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=13648, suffix heap size=147360,new segment heap size=31232  suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=298 2024-11-19T15:16:19,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029439314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029439319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=257, cellSize=11765, totalHeapSize=26968, min timestamp=1732029379311, max timestamp=1732029379324Num uniques 1157; 2024-11-19T15:16:19,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=11765, suffix heap size=148296,new segment heap size=26968  suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=257 2024-11-19T15:16:19,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=257, cellSize=11765, totalHeapSize=26968, min timestamp=1732029379311, max timestamp=1732029379324Num uniques 1157; 2024-11-19T15:16:19,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=11765, suffix heap size=148296,new segment heap size=26968  suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=257 2024-11-19T15:16:19,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=257, cellSize=11765, totalHeapSize=26968, min timestamp=1732029379311, max timestamp=1732029379324Num uniques 1157; 2024-11-19T15:16:19,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=11765, suffix heap size=148296,new segment heap size=26968  suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=257 2024-11-19T15:16:19,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029439361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029439363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1732029379352, max timestamp=1732029379365Num uniques 1116; 2024-11-19T15:16:19,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51106, new segment data size=9886, suffix heap size=144032,new segment heap size=22704  suffix off heap size=0, new segment off heap size=0, suffix cells count=1116, new segment cells count=216 2024-11-19T15:16:19,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1732029379352, max timestamp=1732029379365Num uniques 1116; 2024-11-19T15:16:19,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51106, new segment data size=9886, suffix heap size=144032,new segment heap size=22704  suffix off heap size=0, new segment off heap size=0, suffix cells count=1116, new segment cells count=216 2024-11-19T15:16:19,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1732029379352, max timestamp=1732029379365Num uniques 1116; 2024-11-19T15:16:19,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51106, new segment data size=9886, suffix heap size=144032,new segment heap size=22704  suffix off heap size=0, new segment off heap size=0, suffix cells count=1116, new segment cells count=216 2024-11-19T15:16:19,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:19,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:19,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8004, totalHeapSize=18440, min timestamp=1732029379438, max timestamp=1732029379447Num uniques 1075; 2024-11-19T15:16:19,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49224, new segment data size=8004, suffix heap size=139768,new segment heap size=18440  suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=175 2024-11-19T15:16:19,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8004, totalHeapSize=18440, min timestamp=1732029379438, max timestamp=1732029379447Num uniques 1075; 2024-11-19T15:16:19,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49224, new segment data size=8004, suffix heap size=139768,new segment heap size=18440  suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=175 2024-11-19T15:16:19,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8008, totalHeapSize=18440, min timestamp=1732029379438, max timestamp=1732029379447Num uniques 1075; 2024-11-19T15:16:19,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/17621af7af784cf8a087af60e0091eb2 2024-11-19T15:16:19,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/50570b5b74e7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49228, new segment data size=8008, suffix heap size=139768,new segment heap size=18440  suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=175 2024-11-19T15:16:19,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/db6ac21e756644fbacbfbc373a9c8015 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/db6ac21e756644fbacbfbc373a9c8015 2024-11-19T15:16:19,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/db6ac21e756644fbacbfbc373a9c8015, entries=150, sequenceid=100, filesize=30.2 K 2024-11-19T15:16:19,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/dd75489a178f4dd3b4eb0de69e86d23a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/dd75489a178f4dd3b4eb0de69e86d23a 2024-11-19T15:16:19,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/dd75489a178f4dd3b4eb0de69e86d23a, entries=150, sequenceid=100, filesize=11.7 K 2024-11-19T15:16:19,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/17621af7af784cf8a087af60e0091eb2 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/17621af7af784cf8a087af60e0091eb2 2024-11-19T15:16:19,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/17621af7af784cf8a087af60e0091eb2, entries=150, sequenceid=100, filesize=11.7 K 2024-11-19T15:16:19,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=53.67 KB/54960 for 017a8429196619e4c293e69ec8b529c1 in 1302ms, sequenceid=100, compaction requested=true 2024-11-19T15:16:19,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:19,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:16:19,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:19,490 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:16:19,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:16:19,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:19,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:16:19,490 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:16:19,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:19,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=73.80 KB heapSize=204.19 KB 2024-11-19T15:16:19,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:19,492 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:16:19,492 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/A is initiating minor compaction (all files) 2024-11-19T15:16:19,492 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:16:19,492 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/A in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,492 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/B is initiating minor compaction (all files) 2024-11-19T15:16:19,492 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/B in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,492 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/2a619cf2ff0f4311aea60a19aaca3faf, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/624679725a3e4509aaabb49d4fae5136, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f66c8f995eb2463f9e245f032ee63994, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/db6ac21e756644fbacbfbc373a9c8015] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=120.9 K 2024-11-19T15:16:19,492 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/9f7f9e1cc6084eb2898999ccbec86467, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/1a5f68f1d5564eee9eb476f888df14b0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/feacff36651d4c9c9c9d82e98d23f737, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/dd75489a178f4dd3b4eb0de69e86d23a] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=44.6 K 2024-11-19T15:16:19,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:19,492 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:19,492 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/2a619cf2ff0f4311aea60a19aaca3faf, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/624679725a3e4509aaabb49d4fae5136, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f66c8f995eb2463f9e245f032ee63994, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/db6ac21e756644fbacbfbc373a9c8015] 2024-11-19T15:16:19,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:19,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:19,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T15:16:19,492 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f7f9e1cc6084eb2898999ccbec86467, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732029375624 2024-11-19T15:16:19,493 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a619cf2ff0f4311aea60a19aaca3faf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732029375624 2024-11-19T15:16:19,493 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a5f68f1d5564eee9eb476f888df14b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732029375676 2024-11-19T15:16:19,493 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624679725a3e4509aaabb49d4fae5136, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732029375676 2024-11-19T15:16:19,493 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting feacff36651d4c9c9c9d82e98d23f737, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732029376324 2024-11-19T15:16:19,494 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting f66c8f995eb2463f9e245f032ee63994, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732029376324 2024-11-19T15:16:19,494 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting dd75489a178f4dd3b4eb0de69e86d23a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732029377038 2024-11-19T15:16:19,495 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting db6ac21e756644fbacbfbc373a9c8015, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732029377038 2024-11-19T15:16:19,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196ccfd464419a4fda9d10726e96cfff25_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029379468/Put/seqid=0 2024-11-19T15:16:19,511 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:19,514 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:19,514 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/6c6ca60cd4ec4e4da6736614d72ec675 is 50, key is test_row_0/B:col10/1732029377038/Put/seqid=0 2024-11-19T15:16:19,515 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411192f43476f684040cf8c358ab25c041f44_017a8429196619e4c293e69ec8b529c1 store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:19,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742388_1564 (size=12454) 2024-11-19T15:16:19,518 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411192f43476f684040cf8c358ab25c041f44_017a8429196619e4c293e69ec8b529c1, store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:19,518 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192f43476f684040cf8c358ab25c041f44_017a8429196619e4c293e69ec8b529c1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:19,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742389_1565 (size=12139) 2024-11-19T15:16:19,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029439524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029439526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,535 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/6c6ca60cd4ec4e4da6736614d72ec675 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/6c6ca60cd4ec4e4da6736614d72ec675 2024-11-19T15:16:19,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742390_1566 (size=4469) 2024-11-19T15:16:19,538 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#A#compaction#483 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:19,539 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/4330593e210e4417883f672920e8503e is 175, key is test_row_0/A:col10/1732029377038/Put/seqid=0 2024-11-19T15:16:19,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029439529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,541 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/B of 017a8429196619e4c293e69ec8b529c1 into 6c6ca60cd4ec4e4da6736614d72ec675(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:19,541 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:19,541 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/B, priority=12, startTime=1732029379490; duration=0sec 2024-11-19T15:16:19,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,541 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,541 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:B 2024-11-19T15:16:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029439529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,541 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:16:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029439531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,543 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:16:19,543 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/C is initiating minor compaction (all files) 2024-11-19T15:16:19,543 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/C in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,543 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7de4254f979c416f80b36e8f230eb064, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/4fcfe0996bbd48fc98827e038c1d7da0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/10ad70c2b8e24234b99f9ce70968b883, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/17621af7af784cf8a087af60e0091eb2] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=44.6 K 2024-11-19T15:16:19,543 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7de4254f979c416f80b36e8f230eb064, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732029375624 2024-11-19T15:16:19,544 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fcfe0996bbd48fc98827e038c1d7da0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732029375676 2024-11-19T15:16:19,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742391_1567 (size=31093) 2024-11-19T15:16:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T15:16:19,544 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 10ad70c2b8e24234b99f9ce70968b883, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732029376324 2024-11-19T15:16:19,545 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 17621af7af784cf8a087af60e0091eb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732029377038 2024-11-19T15:16:19,549 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/4330593e210e4417883f672920e8503e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/4330593e210e4417883f672920e8503e 2024-11-19T15:16:19,553 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/A of 017a8429196619e4c293e69ec8b529c1 into 4330593e210e4417883f672920e8503e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:19,553 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:19,553 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/A, priority=12, startTime=1732029379490; duration=0sec 2024-11-19T15:16:19,553 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:19,554 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:A 2024-11-19T15:16:19,554 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#C#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:19,554 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/eeedab8348c5411ca437e60348cc4d20 is 50, key is test_row_0/C:col10/1732029377038/Put/seqid=0 2024-11-19T15:16:19,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742392_1568 (size=12139) 2024-11-19T15:16:19,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029439644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029439644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029439644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029439644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029439644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:19,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:19,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:19,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:19,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029439854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029439855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029439855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029439855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029439855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:19,918 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:19,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196ccfd464419a4fda9d10726e96cfff25_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196ccfd464419a4fda9d10726e96cfff25_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:19,922 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/517924c75f1a4d339522e36431f186bd, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:19,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/517924c75f1a4d339522e36431f186bd is 175, key is test_row_0/A:col10/1732029379468/Put/seqid=0 2024-11-19T15:16:19,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742393_1569 (size=31255) 2024-11-19T15:16:19,961 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/eeedab8348c5411ca437e60348cc4d20 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/eeedab8348c5411ca437e60348cc4d20 2024-11-19T15:16:19,965 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/C of 017a8429196619e4c293e69ec8b529c1 into eeedab8348c5411ca437e60348cc4d20(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:19,965 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:19,965 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/C, priority=12, startTime=1732029379490; duration=0sec 2024-11-19T15:16:19,965 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:19,965 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:C 2024-11-19T15:16:19,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:19,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:19,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:19,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T15:16:20,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:20,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:20,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:20,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:20,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029440164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:20,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029440164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:20,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029440164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:20,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029440174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029440174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:20,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:20,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,326 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=540, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/517924c75f1a4d339522e36431f186bd 2024-11-19T15:16:20,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/7ca6f1a63090450c8cf397072dafb30e is 50, key is test_row_0/B:col10/1732029379468/Put/seqid=0 2024-11-19T15:16:20,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742394_1570 (size=12301) 2024-11-19T15:16:20,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:20,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:20,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:20,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:20,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:20,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:20,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:20,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:20,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029440674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:20,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029440674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:20,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029440674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:20,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:20,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:20,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029440684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:20,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029440694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:20,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/7ca6f1a63090450c8cf397072dafb30e 2024-11-19T15:16:20,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/a25035af17fb42db8f2314dde455d021 is 50, key is test_row_0/C:col10/1732029379468/Put/seqid=0 2024-11-19T15:16:20,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742395_1571 (size=12301) 2024-11-19T15:16:20,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:20,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:20,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:20,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:20,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:20,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:20,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:20,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:20,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:21,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:21,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:21,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:21,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:21,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:21,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T15:16:21,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/a25035af17fb42db8f2314dde455d021 2024-11-19T15:16:21,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/517924c75f1a4d339522e36431f186bd as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/517924c75f1a4d339522e36431f186bd 2024-11-19T15:16:21,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/517924c75f1a4d339522e36431f186bd, entries=150, sequenceid=540, filesize=30.5 K 2024-11-19T15:16:21,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/7ca6f1a63090450c8cf397072dafb30e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/7ca6f1a63090450c8cf397072dafb30e 2024-11-19T15:16:21,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/7ca6f1a63090450c8cf397072dafb30e, entries=150, sequenceid=540, filesize=12.0 K 2024-11-19T15:16:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/a25035af17fb42db8f2314dde455d021 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a25035af17fb42db8f2314dde455d021 2024-11-19T15:16:21,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a25035af17fb42db8f2314dde455d021, entries=150, sequenceid=540, filesize=12.0 K 2024-11-19T15:16:21,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~204.14 KB/209040, currentSize=107.34 KB/109920 for 017a8429196619e4c293e69ec8b529c1 in 1671ms, sequenceid=540, compaction requested=false 2024-11-19T15:16:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:21,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T15:16:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:21,209 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T15:16:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:21,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a8339d06df984eae9edce9982307a9a4_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029379528/Put/seqid=0 2024-11-19T15:16:21,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742396_1572 (size=12454) 2024-11-19T15:16:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:21,624 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a8339d06df984eae9edce9982307a9a4_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a8339d06df984eae9edce9982307a9a4_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:21,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/72a575a2e3f84f37b0a29d7250d6c643, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:21,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/72a575a2e3f84f37b0a29d7250d6c643 is 175, key is test_row_0/A:col10/1732029379528/Put/seqid=0 2024-11-19T15:16:21,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742397_1573 (size=31255) 2024-11-19T15:16:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:21,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:21,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029441706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:21,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029441707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:21,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029441710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:21,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029441711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:21,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029441714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:21,718 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in 017a8429196619e4c293e69ec8b529c1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:16:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:21,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029441814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:21,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:21,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029441824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:21,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029441824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:21,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029441824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:21,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029442024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,029 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=562, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/72a575a2e3f84f37b0a29d7250d6c643 2024-11-19T15:16:22,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029442034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029442034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/5df2fc8d1b9c47c3a0efce5fed52a262 is 50, key is test_row_0/B:col10/1732029379528/Put/seqid=0 2024-11-19T15:16:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029442034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742398_1574 (size=12301) 2024-11-19T15:16:22,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029442334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029442344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029442345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029442345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,439 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/5df2fc8d1b9c47c3a0efce5fed52a262 2024-11-19T15:16:22,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/47bbb83830894c6fbbeada2f7049bfac is 50, key is test_row_0/C:col10/1732029379528/Put/seqid=0 2024-11-19T15:16:22,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742399_1575 (size=12301) 2024-11-19T15:16:22,450 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/47bbb83830894c6fbbeada2f7049bfac 2024-11-19T15:16:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/72a575a2e3f84f37b0a29d7250d6c643 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/72a575a2e3f84f37b0a29d7250d6c643 2024-11-19T15:16:22,456 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/72a575a2e3f84f37b0a29d7250d6c643, entries=150, sequenceid=562, filesize=30.5 K 2024-11-19T15:16:22,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/5df2fc8d1b9c47c3a0efce5fed52a262 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/5df2fc8d1b9c47c3a0efce5fed52a262 2024-11-19T15:16:22,459 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/5df2fc8d1b9c47c3a0efce5fed52a262, entries=150, sequenceid=562, filesize=12.0 K 2024-11-19T15:16:22,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/47bbb83830894c6fbbeada2f7049bfac as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/47bbb83830894c6fbbeada2f7049bfac 2024-11-19T15:16:22,463 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/47bbb83830894c6fbbeada2f7049bfac, entries=150, sequenceid=562, filesize=12.0 K 2024-11-19T15:16:22,464 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 017a8429196619e4c293e69ec8b529c1 in 1255ms, sequenceid=562, compaction requested=true 2024-11-19T15:16:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-19T15:16:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-19T15:16:22,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-19T15:16:22,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5480 sec 2024-11-19T15:16:22,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.5520 sec 2024-11-19T15:16:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:22,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T15:16:22,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:22,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:22,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:22,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:22,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:22,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:22,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193149049707e242ac825d28b29e56a3b4_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029382846/Put/seqid=0 2024-11-19T15:16:22,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742400_1576 (size=14994) 2024-11-19T15:16:22,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029442873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029442874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029442875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029442877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029442984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029442984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029442984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029442984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T15:16:23,074 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:23,075 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:16:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-19T15:16:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T15:16:23,076 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:16:23,077 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:16:23,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:16:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T15:16:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029443194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029443194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029443194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029443194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:23,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T15:16:23,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:23,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:23,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:23,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,265 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:23,268 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193149049707e242ac825d28b29e56a3b4_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193149049707e242ac825d28b29e56a3b4_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:23,269 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/22c2aaea4100492fac7af7cb7df89833, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:23,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/22c2aaea4100492fac7af7cb7df89833 is 175, key is test_row_0/A:col10/1732029382846/Put/seqid=0 2024-11-19T15:16:23,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742401_1577 (size=39949) 2024-11-19T15:16:23,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T15:16:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:23,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T15:16:23,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029443504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:23,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029443504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029443504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:23,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029443504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:23,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T15:16:23,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:23,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:23,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:23,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,673 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=576, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/22c2aaea4100492fac7af7cb7df89833 2024-11-19T15:16:23,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/a1fc4e9240804d5f905f8b5523044689 is 50, key is test_row_0/B:col10/1732029382846/Put/seqid=0 2024-11-19T15:16:23,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742402_1578 (size=12301) 2024-11-19T15:16:23,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T15:16:23,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:23,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:23,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:23,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T15:16:23,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029443734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:23,737 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in 017a8429196619e4c293e69ec8b529c1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:16:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:23,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T15:16:23,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:23,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:23,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:23,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T15:16:23,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:23,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:24,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029444014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:24,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029444014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:24,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029444014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:24,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029444014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:24,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/a1fc4e9240804d5f905f8b5523044689 2024-11-19T15:16:24,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/a47269a1de27436ab71ca4b962726fea is 50, key is test_row_0/C:col10/1732029382846/Put/seqid=0 2024-11-19T15:16:24,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742403_1579 (size=12301) 2024-11-19T15:16:24,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T15:16:24,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:24,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:24,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:24,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:24,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T15:16:24,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T15:16:24,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:24,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:24,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:24,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:24,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:24,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T15:16:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:24,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:24,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:24,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/a47269a1de27436ab71ca4b962726fea 2024-11-19T15:16:24,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/22c2aaea4100492fac7af7cb7df89833 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/22c2aaea4100492fac7af7cb7df89833 2024-11-19T15:16:24,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/22c2aaea4100492fac7af7cb7df89833, entries=200, sequenceid=576, filesize=39.0 K 2024-11-19T15:16:24,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/a1fc4e9240804d5f905f8b5523044689 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a1fc4e9240804d5f905f8b5523044689 2024-11-19T15:16:24,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a1fc4e9240804d5f905f8b5523044689, entries=150, sequenceid=576, filesize=12.0 K 2024-11-19T15:16:24,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/a47269a1de27436ab71ca4b962726fea as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a47269a1de27436ab71ca4b962726fea 2024-11-19T15:16:24,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a47269a1de27436ab71ca4b962726fea, entries=150, sequenceid=576, filesize=12.0 K 2024-11-19T15:16:24,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 017a8429196619e4c293e69ec8b529c1 in 1661ms, sequenceid=576, compaction requested=true 2024-11-19T15:16:24,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:16:24,510 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:16:24,510 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:16:24,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:24,511 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:16:24,511 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49042 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:16:24,511 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/A is initiating minor compaction (all files) 2024-11-19T15:16:24,511 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/B is initiating minor compaction (all files) 2024-11-19T15:16:24,511 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/A in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:24,511 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/B in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:24,511 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/4330593e210e4417883f672920e8503e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/517924c75f1a4d339522e36431f186bd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/72a575a2e3f84f37b0a29d7250d6c643, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/22c2aaea4100492fac7af7cb7df89833] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=130.4 K 2024-11-19T15:16:24,511 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/6c6ca60cd4ec4e4da6736614d72ec675, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/7ca6f1a63090450c8cf397072dafb30e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/5df2fc8d1b9c47c3a0efce5fed52a262, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a1fc4e9240804d5f905f8b5523044689] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=47.9 K 2024-11-19T15:16:24,511 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:24,511 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/4330593e210e4417883f672920e8503e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/517924c75f1a4d339522e36431f186bd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/72a575a2e3f84f37b0a29d7250d6c643, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/22c2aaea4100492fac7af7cb7df89833] 2024-11-19T15:16:24,511 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c6ca60cd4ec4e4da6736614d72ec675, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732029377038 2024-11-19T15:16:24,511 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4330593e210e4417883f672920e8503e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732029377038 2024-11-19T15:16:24,512 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca6f1a63090450c8cf397072dafb30e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732029379447 2024-11-19T15:16:24,512 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 517924c75f1a4d339522e36431f186bd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732029379447 2024-11-19T15:16:24,512 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 5df2fc8d1b9c47c3a0efce5fed52a262, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1732029379525 2024-11-19T15:16:24,512 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting a1fc4e9240804d5f905f8b5523044689, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1732029381705 2024-11-19T15:16:24,512 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72a575a2e3f84f37b0a29d7250d6c643, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1732029379525 2024-11-19T15:16:24,512 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22c2aaea4100492fac7af7cb7df89833, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1732029381705 2024-11-19T15:16:24,531 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#B#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:24,531 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/629796579b6c426285c76eae3ca3f464 is 50, key is test_row_0/B:col10/1732029382846/Put/seqid=0 2024-11-19T15:16:24,532 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:24,537 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411199c5eb2b37db241adb7c6b279595b4f78_017a8429196619e4c293e69ec8b529c1 store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:24,540 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411199c5eb2b37db241adb7c6b279595b4f78_017a8429196619e4c293e69ec8b529c1, store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:24,540 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199c5eb2b37db241adb7c6b279595b4f78_017a8429196619e4c293e69ec8b529c1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:24,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742404_1580 (size=12575) 2024-11-19T15:16:24,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742405_1581 (size=4469) 2024-11-19T15:16:24,548 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#A#compaction#495 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:24,548 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/87633ad6378c4d3f873c0134c631c7be is 175, key is test_row_0/A:col10/1732029382846/Put/seqid=0 2024-11-19T15:16:24,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742406_1582 (size=31529) 2024-11-19T15:16:24,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T15:16:24,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:24,605 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T15:16:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:24,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196b0037bcddc8407ca70e5716cb14a64a_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029382875/Put/seqid=0 2024-11-19T15:16:24,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742407_1583 (size=12454) 2024-11-19T15:16:24,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:24,620 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196b0037bcddc8407ca70e5716cb14a64a_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196b0037bcddc8407ca70e5716cb14a64a_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/0568911062dd40ff8e75e7c000e0fd02, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:24,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/0568911062dd40ff8e75e7c000e0fd02 is 175, key is test_row_0/A:col10/1732029382875/Put/seqid=0 2024-11-19T15:16:24,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742408_1584 (size=31255) 2024-11-19T15:16:24,950 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/629796579b6c426285c76eae3ca3f464 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/629796579b6c426285c76eae3ca3f464 2024-11-19T15:16:24,965 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/B of 017a8429196619e4c293e69ec8b529c1 into 629796579b6c426285c76eae3ca3f464(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:24,965 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:24,965 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/B, priority=12, startTime=1732029384510; duration=0sec 2024-11-19T15:16:24,965 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:24,965 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:B 2024-11-19T15:16:24,965 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T15:16:24,967 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49042 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T15:16:24,967 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/C is initiating minor compaction (all files) 2024-11-19T15:16:24,967 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/C in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:24,967 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/87633ad6378c4d3f873c0134c631c7be as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/87633ad6378c4d3f873c0134c631c7be 2024-11-19T15:16:24,967 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/eeedab8348c5411ca437e60348cc4d20, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a25035af17fb42db8f2314dde455d021, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/47bbb83830894c6fbbeada2f7049bfac, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a47269a1de27436ab71ca4b962726fea] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=47.9 K 2024-11-19T15:16:24,967 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting eeedab8348c5411ca437e60348cc4d20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732029377038 2024-11-19T15:16:24,967 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting a25035af17fb42db8f2314dde455d021, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1732029379447 2024-11-19T15:16:24,968 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 47bbb83830894c6fbbeada2f7049bfac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1732029379525 2024-11-19T15:16:24,968 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting a47269a1de27436ab71ca4b962726fea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1732029381705 2024-11-19T15:16:24,971 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/A of 017a8429196619e4c293e69ec8b529c1 into 87633ad6378c4d3f873c0134c631c7be(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:24,971 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:24,971 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/A, priority=12, startTime=1732029384509; duration=0sec 2024-11-19T15:16:24,971 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:24,971 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:A 2024-11-19T15:16:24,976 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:24,976 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/d330aa1297984538a79291682b4ca624 is 50, key is test_row_0/C:col10/1732029382846/Put/seqid=0 2024-11-19T15:16:24,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742409_1585 (size=12575) 2024-11-19T15:16:24,987 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/d330aa1297984538a79291682b4ca624 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/d330aa1297984538a79291682b4ca624 2024-11-19T15:16:24,993 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/C of 017a8429196619e4c293e69ec8b529c1 into d330aa1297984538a79291682b4ca624(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:24,993 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:24,993 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/C, priority=12, startTime=1732029384510; duration=0sec 2024-11-19T15:16:24,993 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:24,993 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:C 2024-11-19T15:16:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:25,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:25,025 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=594, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/0568911062dd40ff8e75e7c000e0fd02 2024-11-19T15:16:25,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/d73c60b51ad64fa5b42a9b49e7b28981 is 50, key is test_row_0/B:col10/1732029382875/Put/seqid=0 2024-11-19T15:16:25,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742410_1586 (size=12301) 2024-11-19T15:16:25,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029445040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029445042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029445041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029445044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029445154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029445154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029445154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029445154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T15:16:25,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029445364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029445364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029445364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029445364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,438 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/d73c60b51ad64fa5b42a9b49e7b28981 2024-11-19T15:16:25,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/d431b3db20894a468191a0a675c822e8 is 50, key is test_row_0/C:col10/1732029382875/Put/seqid=0 2024-11-19T15:16:25,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742411_1587 (size=12301) 2024-11-19T15:16:25,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029445674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029445675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029445675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029445675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:25,848 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/d431b3db20894a468191a0a675c822e8 2024-11-19T15:16:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/0568911062dd40ff8e75e7c000e0fd02 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/0568911062dd40ff8e75e7c000e0fd02 2024-11-19T15:16:25,855 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/0568911062dd40ff8e75e7c000e0fd02, entries=150, sequenceid=594, filesize=30.5 K 2024-11-19T15:16:25,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/d73c60b51ad64fa5b42a9b49e7b28981 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d73c60b51ad64fa5b42a9b49e7b28981 2024-11-19T15:16:25,858 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d73c60b51ad64fa5b42a9b49e7b28981, entries=150, sequenceid=594, filesize=12.0 K 2024-11-19T15:16:25,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/d431b3db20894a468191a0a675c822e8 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/d431b3db20894a468191a0a675c822e8 2024-11-19T15:16:25,861 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/d431b3db20894a468191a0a675c822e8, entries=150, sequenceid=594, filesize=12.0 K 2024-11-19T15:16:25,862 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 017a8429196619e4c293e69ec8b529c1 in 1257ms, sequenceid=594, compaction requested=false 2024-11-19T15:16:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-19T15:16:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-19T15:16:25,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-19T15:16:25,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7860 sec 2024-11-19T15:16:25,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.7900 sec 2024-11-19T15:16:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:26,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T15:16:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:26,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111983872b16796045388b54d21c51a833ac_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029386186/Put/seqid=0 2024-11-19T15:16:26,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742412_1588 (size=12454) 2024-11-19T15:16:26,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029446202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029446204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029446206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029446210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029446314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029446314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029446314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029446324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029446524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029446524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029446524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029446534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,598 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:26,601 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111983872b16796045388b54d21c51a833ac_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111983872b16796045388b54d21c51a833ac_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:26,602 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/d1b4cc88126440dd8802ec6236ea4655, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:26,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/d1b4cc88126440dd8802ec6236ea4655 is 175, key is test_row_0/A:col10/1732029386186/Put/seqid=0 2024-11-19T15:16:26,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742413_1589 (size=31255) 2024-11-19T15:16:26,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029446834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029446834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029446834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:26,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029446844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:27,006 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=613, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/d1b4cc88126440dd8802ec6236ea4655 2024-11-19T15:16:27,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/3cabd70e41b9443e92322a94a0e3722c is 50, key is test_row_0/B:col10/1732029386186/Put/seqid=0 2024-11-19T15:16:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742414_1590 (size=12301) 2024-11-19T15:16:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T15:16:27,234 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:27,235 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:16:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-19T15:16:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T15:16:27,236 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:16:27,237 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:16:27,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:16:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T15:16:27,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029447344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:27,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029447344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:27,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029447344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:27,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029447354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:27,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T15:16:27,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:27,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:27,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:27,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:27,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:27,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/3cabd70e41b9443e92322a94a0e3722c 2024-11-19T15:16:27,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/f8c8b873c357474e80cc56ececc5c0d9 is 50, key is test_row_0/C:col10/1732029386186/Put/seqid=0 2024-11-19T15:16:27,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742415_1591 (size=12301) 2024-11-19T15:16:27,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T15:16:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:27,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T15:16:27,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T15:16:27,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:27,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:27,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:27,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:27,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:27,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53362 deadline: 1732029447784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:27,788 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 50570b5b74e7,46557,1732029218639 for 'test_row_0' in 017a8429196619e4c293e69ec8b529c1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T15:16:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:27,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:27,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/f8c8b873c357474e80cc56ececc5c0d9 2024-11-19T15:16:27,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/d1b4cc88126440dd8802ec6236ea4655 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d1b4cc88126440dd8802ec6236ea4655 2024-11-19T15:16:27,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d1b4cc88126440dd8802ec6236ea4655, entries=150, sequenceid=613, filesize=30.5 K 2024-11-19T15:16:27,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/3cabd70e41b9443e92322a94a0e3722c as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3cabd70e41b9443e92322a94a0e3722c 2024-11-19T15:16:27,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3cabd70e41b9443e92322a94a0e3722c, entries=150, sequenceid=613, filesize=12.0 K 2024-11-19T15:16:27,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/f8c8b873c357474e80cc56ececc5c0d9 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/f8c8b873c357474e80cc56ececc5c0d9 2024-11-19T15:16:27,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/f8c8b873c357474e80cc56ececc5c0d9, entries=150, sequenceid=613, filesize=12.0 K 2024-11-19T15:16:27,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 017a8429196619e4c293e69ec8b529c1 in 1656ms, sequenceid=613, compaction requested=true 2024-11-19T15:16:27,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:27,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:16:27,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:27,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:16:27,842 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:16:27,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:27,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:16:27,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:27,842 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:16:27,843 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:16:27,843 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37177 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:16:27,843 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/A is initiating minor compaction (all files) 2024-11-19T15:16:27,843 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/B is initiating minor compaction (all files) 2024-11-19T15:16:27,843 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/A in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:27,843 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/B in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:27,843 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/87633ad6378c4d3f873c0134c631c7be, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/0568911062dd40ff8e75e7c000e0fd02, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d1b4cc88126440dd8802ec6236ea4655] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=91.8 K 2024-11-19T15:16:27,843 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/629796579b6c426285c76eae3ca3f464, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d73c60b51ad64fa5b42a9b49e7b28981, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3cabd70e41b9443e92322a94a0e3722c] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=36.3 K 2024-11-19T15:16:27,843 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:27,843 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/87633ad6378c4d3f873c0134c631c7be, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/0568911062dd40ff8e75e7c000e0fd02, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d1b4cc88126440dd8802ec6236ea4655] 2024-11-19T15:16:27,844 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 629796579b6c426285c76eae3ca3f464, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1732029381705 2024-11-19T15:16:27,844 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87633ad6378c4d3f873c0134c631c7be, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1732029381705 2024-11-19T15:16:27,844 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0568911062dd40ff8e75e7c000e0fd02, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1732029382870 2024-11-19T15:16:27,844 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d73c60b51ad64fa5b42a9b49e7b28981, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1732029382870 2024-11-19T15:16:27,844 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b4cc88126440dd8802ec6236ea4655, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1732029385042 2024-11-19T15:16:27,844 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cabd70e41b9443e92322a94a0e3722c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1732029385042 2024-11-19T15:16:27,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T15:16:27,849 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:27,849 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T15:16:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:27,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:27,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:27,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:27,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:27,851 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#B#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:27,851 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111963298c7573614931871ebffba455834f_017a8429196619e4c293e69ec8b529c1 store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:27,851 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/61c1d1d0572f42c1914b5dce6479a09a is 50, key is test_row_0/B:col10/1732029386186/Put/seqid=0 2024-11-19T15:16:27,854 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111963298c7573614931871ebffba455834f_017a8429196619e4c293e69ec8b529c1, store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:27,854 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111963298c7573614931871ebffba455834f_017a8429196619e4c293e69ec8b529c1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:27,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742417_1593 (size=4469) 2024-11-19T15:16:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T15:16:27,864 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#A#compaction#503 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:27,865 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/d5e7359a99fe48098d96324c708190af is 175, key is test_row_0/A:col10/1732029386186/Put/seqid=0 2024-11-19T15:16:27,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119adcf1a55b6c648ca97e01a2306b6a42f_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029386199/Put/seqid=0 2024-11-19T15:16:27,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742416_1592 (size=12677) 2024-11-19T15:16:27,887 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/61c1d1d0572f42c1914b5dce6479a09a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/61c1d1d0572f42c1914b5dce6479a09a 2024-11-19T15:16:27,892 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/B of 017a8429196619e4c293e69ec8b529c1 into 61c1d1d0572f42c1914b5dce6479a09a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:27,892 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:27,893 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/B, priority=13, startTime=1732029387842; duration=0sec 2024-11-19T15:16:27,893 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:27,893 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:B 2024-11-19T15:16:27,893 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:16:27,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742418_1594 (size=31631) 2024-11-19T15:16:27,894 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37177 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:16:27,895 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/C is initiating minor compaction (all files) 2024-11-19T15:16:27,895 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/C in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:27,895 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/d330aa1297984538a79291682b4ca624, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/d431b3db20894a468191a0a675c822e8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/f8c8b873c357474e80cc56ececc5c0d9] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=36.3 K 2024-11-19T15:16:27,895 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d330aa1297984538a79291682b4ca624, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1732029381705 2024-11-19T15:16:27,896 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d431b3db20894a468191a0a675c822e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=594, earliestPutTs=1732029382870 2024-11-19T15:16:27,896 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting f8c8b873c357474e80cc56ececc5c0d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1732029385042 2024-11-19T15:16:27,899 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/d5e7359a99fe48098d96324c708190af as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d5e7359a99fe48098d96324c708190af 2024-11-19T15:16:27,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742419_1595 (size=12454) 2024-11-19T15:16:27,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:27,906 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/A of 017a8429196619e4c293e69ec8b529c1 into d5e7359a99fe48098d96324c708190af(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:27,906 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:27,906 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/A, priority=13, startTime=1732029387842; duration=0sec 2024-11-19T15:16:27,906 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:27,906 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:A 2024-11-19T15:16:27,909 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119adcf1a55b6c648ca97e01a2306b6a42f_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119adcf1a55b6c648ca97e01a2306b6a42f_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:27,909 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:27,910 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/9831c5c5d105492d801064af1b5a6ab7 is 50, key is test_row_0/C:col10/1732029386186/Put/seqid=0 2024-11-19T15:16:27,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/09837b0e7265479e815fbe7a6a496ce0, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:27,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/09837b0e7265479e815fbe7a6a496ce0 is 175, key is test_row_0/A:col10/1732029386199/Put/seqid=0 2024-11-19T15:16:27,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742420_1596 (size=12677) 2024-11-19T15:16:27,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742421_1597 (size=31255) 2024-11-19T15:16:27,932 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/9831c5c5d105492d801064af1b5a6ab7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/9831c5c5d105492d801064af1b5a6ab7 2024-11-19T15:16:27,937 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/C of 017a8429196619e4c293e69ec8b529c1 into 9831c5c5d105492d801064af1b5a6ab7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:27,937 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:27,937 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/C, priority=13, startTime=1732029387842; duration=0sec 2024-11-19T15:16:27,937 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:27,937 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:C 2024-11-19T15:16:28,332 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=629, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/09837b0e7265479e815fbe7a6a496ce0 2024-11-19T15:16:28,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/3b49389e45be4b068249841e94ff1bbf is 50, key is test_row_0/B:col10/1732029386199/Put/seqid=0 2024-11-19T15:16:28,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742422_1598 (size=12301) 2024-11-19T15:16:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:28,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T15:16:28,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029448372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:28,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029448373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:28,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:28,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029448374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:28,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029448374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029448484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029448484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029448484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029448485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:28,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029448694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:28,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029448694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029448694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:28,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029448695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:28,743 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/3b49389e45be4b068249841e94ff1bbf 2024-11-19T15:16:28,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/6783cf2e0c604e48a2db1fce7c558d01 is 50, key is test_row_0/C:col10/1732029386199/Put/seqid=0 2024-11-19T15:16:28,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742423_1599 (size=12301) 2024-11-19T15:16:29,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029449004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029449004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029449004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029449004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,155 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/6783cf2e0c604e48a2db1fce7c558d01 2024-11-19T15:16:29,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/09837b0e7265479e815fbe7a6a496ce0 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/09837b0e7265479e815fbe7a6a496ce0 2024-11-19T15:16:29,162 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/09837b0e7265479e815fbe7a6a496ce0, entries=150, sequenceid=629, filesize=30.5 K 2024-11-19T15:16:29,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/3b49389e45be4b068249841e94ff1bbf as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3b49389e45be4b068249841e94ff1bbf 2024-11-19T15:16:29,166 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3b49389e45be4b068249841e94ff1bbf, entries=150, sequenceid=629, filesize=12.0 K 2024-11-19T15:16:29,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/6783cf2e0c604e48a2db1fce7c558d01 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/6783cf2e0c604e48a2db1fce7c558d01 2024-11-19T15:16:29,169 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/6783cf2e0c604e48a2db1fce7c558d01, entries=150, sequenceid=629, filesize=12.0 K 2024-11-19T15:16:29,170 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 017a8429196619e4c293e69ec8b529c1 in 1321ms, sequenceid=629, compaction requested=false 2024-11-19T15:16:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-19T15:16:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-19T15:16:29,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-19T15:16:29,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9340 sec 2024-11-19T15:16:29,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.9380 sec 2024-11-19T15:16:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T15:16:29,384 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:29,385 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:16:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-19T15:16:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T15:16:29,387 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:16:29,387 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:16:29,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:16:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T15:16:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:29,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T15:16:29,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:29,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:29,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:29,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:29,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:29,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:29,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a706a85f596b4c318e2c6d26ec4763a1_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029388371/Put/seqid=0 2024-11-19T15:16:29,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742424_1600 (size=12454) 2024-11-19T15:16:29,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029449527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029449530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029449531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029449532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T15:16:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:29,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:29,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029449634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029449644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029449644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029449644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T15:16:29,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:29,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T15:16:29,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T15:16:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:29,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:29,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:29,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029449844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029449854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029449854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029449855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:29,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:29,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:29,927 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:29,930 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a706a85f596b4c318e2c6d26ec4763a1_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a706a85f596b4c318e2c6d26ec4763a1_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:29,931 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/f167e5e24c9042479b451ea1826dbf9d, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:29,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/f167e5e24c9042479b451ea1826dbf9d is 175, key is test_row_0/A:col10/1732029388371/Put/seqid=0 2024-11-19T15:16:29,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742425_1601 (size=31255) 2024-11-19T15:16:29,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=650, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/f167e5e24c9042479b451ea1826dbf9d 2024-11-19T15:16:29,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/f1f83df1b6114a27ac6e2a884603c78a is 50, key is test_row_0/B:col10/1732029388371/Put/seqid=0 2024-11-19T15:16:29,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742426_1602 (size=12301) 2024-11-19T15:16:29,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/f1f83df1b6114a27ac6e2a884603c78a 2024-11-19T15:16:29,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/f5680694c07c4388ae1a4d969f982eb5 is 50, key is test_row_0/C:col10/1732029388371/Put/seqid=0 2024-11-19T15:16:29,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742427_1603 (size=12301) 2024-11-19T15:16:29,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T15:16:29,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:29,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:29,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:29,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:29,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T15:16:30,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T15:16:30,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:30,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:30,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:30,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:30,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:30,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029450154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:30,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029450164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029450164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029450164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:30,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T15:16:30,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:30,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:30,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:30,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:30,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T15:16:30,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/f5680694c07c4388ae1a4d969f982eb5 2024-11-19T15:16:30,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/f167e5e24c9042479b451ea1826dbf9d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f167e5e24c9042479b451ea1826dbf9d 2024-11-19T15:16:30,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f167e5e24c9042479b451ea1826dbf9d, entries=150, sequenceid=650, filesize=30.5 K 2024-11-19T15:16:30,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/f1f83df1b6114a27ac6e2a884603c78a as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/f1f83df1b6114a27ac6e2a884603c78a 2024-11-19T15:16:30,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/f1f83df1b6114a27ac6e2a884603c78a, entries=150, sequenceid=650, filesize=12.0 K 2024-11-19T15:16:30,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/f5680694c07c4388ae1a4d969f982eb5 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/f5680694c07c4388ae1a4d969f982eb5 2024-11-19T15:16:30,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/f5680694c07c4388ae1a4d969f982eb5, entries=150, sequenceid=650, filesize=12.0 K 2024-11-19T15:16:30,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 017a8429196619e4c293e69ec8b529c1 in 872ms, sequenceid=650, compaction requested=true 2024-11-19T15:16:30,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:30,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:16:30,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:30,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:16:30,389 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:16:30,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:30,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:16:30,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:30,389 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:16:30,390 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:16:30,390 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/A is initiating minor compaction (all files) 2024-11-19T15:16:30,390 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/A in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:30,390 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d5e7359a99fe48098d96324c708190af, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/09837b0e7265479e815fbe7a6a496ce0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f167e5e24c9042479b451ea1826dbf9d] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=91.9 K 2024-11-19T15:16:30,390 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:30,390 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d5e7359a99fe48098d96324c708190af, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/09837b0e7265479e815fbe7a6a496ce0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f167e5e24c9042479b451ea1826dbf9d] 2024-11-19T15:16:30,390 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:16:30,390 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/B is initiating minor compaction (all files) 2024-11-19T15:16:30,390 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/B in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:30,391 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/61c1d1d0572f42c1914b5dce6479a09a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3b49389e45be4b068249841e94ff1bbf, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/f1f83df1b6114a27ac6e2a884603c78a] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=36.4 K 2024-11-19T15:16:30,391 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5e7359a99fe48098d96324c708190af, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1732029385042 2024-11-19T15:16:30,391 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 61c1d1d0572f42c1914b5dce6479a09a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1732029385042 2024-11-19T15:16:30,391 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09837b0e7265479e815fbe7a6a496ce0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1732029386199 2024-11-19T15:16:30,391 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b49389e45be4b068249841e94ff1bbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1732029386199 2024-11-19T15:16:30,392 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f83df1b6114a27ac6e2a884603c78a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732029388367 2024-11-19T15:16:30,392 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting f167e5e24c9042479b451ea1826dbf9d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732029388367 2024-11-19T15:16:30,398 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:30,400 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#B#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:30,401 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/d39a0a949a9e4eb5a80fb2e864eee15f is 50, key is test_row_0/B:col10/1732029388371/Put/seqid=0 2024-11-19T15:16:30,405 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119329875f4860240a7857e6419ce484a36_017a8429196619e4c293e69ec8b529c1 store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:30,407 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119329875f4860240a7857e6419ce484a36_017a8429196619e4c293e69ec8b529c1, store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:30,407 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119329875f4860240a7857e6419ce484a36_017a8429196619e4c293e69ec8b529c1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:30,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742428_1604 (size=12779) 2024-11-19T15:16:30,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742429_1605 (size=4469) 2024-11-19T15:16:30,433 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#A#compaction#512 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:30,434 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/7ea217b4194c4d659e4b63bc0895cb48 is 175, key is test_row_0/A:col10/1732029388371/Put/seqid=0 2024-11-19T15:16:30,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742430_1606 (size=31733) 2024-11-19T15:16:30,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T15:16:30,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:30,457 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T15:16:30,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:30,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:30,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:30,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:30,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:30,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:30,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193c15873f88154f49933110044930e606_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029389530/Put/seqid=0 2024-11-19T15:16:30,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742431_1607 (size=12454) 2024-11-19T15:16:30,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:30,471 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193c15873f88154f49933110044930e606_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193c15873f88154f49933110044930e606_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:30,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/b3dc4048e7824a84a29b2723b10797f3, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/b3dc4048e7824a84a29b2723b10797f3 is 175, key is test_row_0/A:col10/1732029389530/Put/seqid=0 2024-11-19T15:16:30,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742432_1608 (size=31255) 2024-11-19T15:16:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T15:16:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:30,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:30,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029450701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:30,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029450710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:30,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029450714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:30,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029450716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:30,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029450824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:30,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029450824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:30,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029450824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029450824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:30,838 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/d39a0a949a9e4eb5a80fb2e864eee15f as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d39a0a949a9e4eb5a80fb2e864eee15f 2024-11-19T15:16:30,841 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/7ea217b4194c4d659e4b63bc0895cb48 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/7ea217b4194c4d659e4b63bc0895cb48 2024-11-19T15:16:30,843 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/B of 017a8429196619e4c293e69ec8b529c1 into d39a0a949a9e4eb5a80fb2e864eee15f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:30,843 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:30,843 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/B, priority=13, startTime=1732029390389; duration=0sec 2024-11-19T15:16:30,843 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:30,843 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:B 2024-11-19T15:16:30,843 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T15:16:30,846 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T15:16:30,846 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/C is initiating minor compaction (all files) 2024-11-19T15:16:30,846 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/C in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:30,846 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/9831c5c5d105492d801064af1b5a6ab7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/6783cf2e0c604e48a2db1fce7c558d01, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/f5680694c07c4388ae1a4d969f982eb5] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=36.4 K 2024-11-19T15:16:30,848 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 9831c5c5d105492d801064af1b5a6ab7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1732029385042 2024-11-19T15:16:30,849 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/A of 017a8429196619e4c293e69ec8b529c1 into 7ea217b4194c4d659e4b63bc0895cb48(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:30,849 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:30,849 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/A, priority=13, startTime=1732029390389; duration=0sec 2024-11-19T15:16:30,849 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:30,849 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:A 2024-11-19T15:16:30,849 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 6783cf2e0c604e48a2db1fce7c558d01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1732029386199 2024-11-19T15:16:30,850 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting f5680694c07c4388ae1a4d969f982eb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732029388367 2024-11-19T15:16:30,861 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#C#compaction#515 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:30,862 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/448d24d43b1a4085a93a9bf9a57c37a4 is 50, key is test_row_0/C:col10/1732029388371/Put/seqid=0 2024-11-19T15:16:30,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742433_1609 (size=12779) 2024-11-19T15:16:30,876 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/448d24d43b1a4085a93a9bf9a57c37a4 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/448d24d43b1a4085a93a9bf9a57c37a4 2024-11-19T15:16:30,876 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=664, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/b3dc4048e7824a84a29b2723b10797f3 2024-11-19T15:16:30,881 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/C of 017a8429196619e4c293e69ec8b529c1 into 448d24d43b1a4085a93a9bf9a57c37a4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:30,881 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:30,881 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/C, priority=13, startTime=1732029390389; duration=0sec 2024-11-19T15:16:30,881 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:30,881 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:C 2024-11-19T15:16:30,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/327078dd0b80462daf8126086936a30d is 50, key is test_row_0/B:col10/1732029389530/Put/seqid=0 2024-11-19T15:16:30,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742434_1610 (size=12301) 2024-11-19T15:16:30,914 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/327078dd0b80462daf8126086936a30d 2024-11-19T15:16:30,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/17e94e38e35740299ec981e7812ccc00 is 50, key is test_row_0/C:col10/1732029389530/Put/seqid=0 2024-11-19T15:16:30,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742435_1611 (size=12301) 2024-11-19T15:16:31,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029451034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:31,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029451034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:31,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029451034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:31,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029451034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:31,339 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/17e94e38e35740299ec981e7812ccc00 2024-11-19T15:16:31,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/b3dc4048e7824a84a29b2723b10797f3 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/b3dc4048e7824a84a29b2723b10797f3 2024-11-19T15:16:31,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:31,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029451344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:31,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029451345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:31,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:31,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029451345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:31,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:31,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029451345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:31,350 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/b3dc4048e7824a84a29b2723b10797f3, entries=150, sequenceid=664, filesize=30.5 K 2024-11-19T15:16:31,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/327078dd0b80462daf8126086936a30d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/327078dd0b80462daf8126086936a30d 2024-11-19T15:16:31,354 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/327078dd0b80462daf8126086936a30d, entries=150, sequenceid=664, filesize=12.0 K 2024-11-19T15:16:31,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/17e94e38e35740299ec981e7812ccc00 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/17e94e38e35740299ec981e7812ccc00 2024-11-19T15:16:31,358 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/17e94e38e35740299ec981e7812ccc00, entries=150, sequenceid=664, filesize=12.0 K 2024-11-19T15:16:31,359 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 017a8429196619e4c293e69ec8b529c1 in 903ms, sequenceid=664, compaction requested=false 2024-11-19T15:16:31,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:31,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:31,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-19T15:16:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-19T15:16:31,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-19T15:16:31,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9730 sec 2024-11-19T15:16:31,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.9770 sec 2024-11-19T15:16:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T15:16:31,534 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:31,534 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:16:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-19T15:16:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T15:16:31,536 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:16:31,537 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:16:31,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:16:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T15:16:31,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T15:16:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:31,690 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T15:16:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:31,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111994750fff398f4cde86fe840eeb5cd9e8_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029390698/Put/seqid=0 2024-11-19T15:16:31,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742436_1612 (size=12454) 2024-11-19T15:16:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:31,736 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111994750fff398f4cde86fe840eeb5cd9e8_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111994750fff398f4cde86fe840eeb5cd9e8_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:31,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/e45a357bfdb74cfaa02275d9f8ae2bb7, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:31,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/e45a357bfdb74cfaa02275d9f8ae2bb7 is 175, key is test_row_0/A:col10/1732029390698/Put/seqid=0 2024-11-19T15:16:31,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742437_1613 (size=31255) 2024-11-19T15:16:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T15:16:31,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:31,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:31,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029451869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:31,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029451902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:31,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029451902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:31,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029451902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029452004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029452014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029452014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029452014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,156 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=686, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/e45a357bfdb74cfaa02275d9f8ae2bb7 2024-11-19T15:16:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T15:16:32,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/88925e91ed044934988fdbdd8fe298b8 is 50, key is test_row_0/B:col10/1732029390698/Put/seqid=0 2024-11-19T15:16:32,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742438_1614 (size=12301) 2024-11-19T15:16:32,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029452214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:32,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029452224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:32,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029452224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:32,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029452224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029452525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029452534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:32,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:32,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029452534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:32,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029452534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:32,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:32,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:32,578 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=686 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/88925e91ed044934988fdbdd8fe298b8 2024-11-19T15:16:32,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/7e83730b046147b38ba7607323a93644 is 50, key is test_row_0/C:col10/1732029390698/Put/seqid=0 2024-11-19T15:16:32,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742439_1615 (size=12301) 2024-11-19T15:16:32,595 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=686 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/7e83730b046147b38ba7607323a93644 2024-11-19T15:16:32,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/e45a357bfdb74cfaa02275d9f8ae2bb7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/e45a357bfdb74cfaa02275d9f8ae2bb7 2024-11-19T15:16:32,602 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/e45a357bfdb74cfaa02275d9f8ae2bb7, entries=150, sequenceid=686, filesize=30.5 K 2024-11-19T15:16:32,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/88925e91ed044934988fdbdd8fe298b8 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/88925e91ed044934988fdbdd8fe298b8 2024-11-19T15:16:32,606 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/88925e91ed044934988fdbdd8fe298b8, entries=150, sequenceid=686, filesize=12.0 K 2024-11-19T15:16:32,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/7e83730b046147b38ba7607323a93644 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7e83730b046147b38ba7607323a93644 2024-11-19T15:16:32,611 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7e83730b046147b38ba7607323a93644, entries=150, sequenceid=686, filesize=12.0 K 2024-11-19T15:16:32,612 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 017a8429196619e4c293e69ec8b529c1 in 922ms, sequenceid=686, compaction requested=true 2024-11-19T15:16:32,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:32,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:32,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-19T15:16:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-19T15:16:32,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-19T15:16:32,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0770 sec 2024-11-19T15:16:32,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.0830 sec 2024-11-19T15:16:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T15:16:32,674 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:32,675 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:16:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-19T15:16:32,676 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:16:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T15:16:32,678 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:16:32,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:16:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T15:16:32,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-19T15:16:32,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:32,830 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T15:16:32,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:32,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:32,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:32,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:32,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:32,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:32,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b94208ba8a1145868c71715279cc3610_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029391872/Put/seqid=0 2024-11-19T15:16:32,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742440_1616 (size=12454) 2024-11-19T15:16:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T15:16:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:33,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:33,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029453060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029453063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029453064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029453066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029453174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029453176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029453176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029453177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:33,263 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b94208ba8a1145868c71715279cc3610_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b94208ba8a1145868c71715279cc3610_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:33,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/f247d597d06a49a2b57fcbe8c7c10e02, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:33,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/f247d597d06a49a2b57fcbe8c7c10e02 is 175, key is test_row_0/A:col10/1732029391872/Put/seqid=0 2024-11-19T15:16:33,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742441_1617 (size=31255) 2024-11-19T15:16:33,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T15:16:33,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029453384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029453385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029453394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029453394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,685 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=700, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/f247d597d06a49a2b57fcbe8c7c10e02 2024-11-19T15:16:33,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/a09fbf3da695414ead87f45f7723f7fe is 50, key is test_row_0/B:col10/1732029391872/Put/seqid=0 2024-11-19T15:16:33,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029453698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029453698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029453706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742442_1618 (size=12301) 2024-11-19T15:16:33,711 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=700 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/a09fbf3da695414ead87f45f7723f7fe 2024-11-19T15:16:33,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029453709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:33,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:33,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:33,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/16ed5e088c4b4492b7ac5f83a038bebd is 50, key is test_row_0/C:col10/1732029391872/Put/seqid=0 2024-11-19T15:16:33,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742443_1619 (size=12301) 2024-11-19T15:16:33,723 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=700 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/16ed5e088c4b4492b7ac5f83a038bebd 2024-11-19T15:16:33,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/f247d597d06a49a2b57fcbe8c7c10e02 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f247d597d06a49a2b57fcbe8c7c10e02 2024-11-19T15:16:33,747 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f247d597d06a49a2b57fcbe8c7c10e02, entries=150, sequenceid=700, filesize=30.5 K 2024-11-19T15:16:33,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/a09fbf3da695414ead87f45f7723f7fe as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a09fbf3da695414ead87f45f7723f7fe 2024-11-19T15:16:33,758 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a09fbf3da695414ead87f45f7723f7fe, entries=150, sequenceid=700, filesize=12.0 K 2024-11-19T15:16:33,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/16ed5e088c4b4492b7ac5f83a038bebd as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/16ed5e088c4b4492b7ac5f83a038bebd 2024-11-19T15:16:33,767 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/16ed5e088c4b4492b7ac5f83a038bebd, entries=150, sequenceid=700, filesize=12.0 K 2024-11-19T15:16:33,768 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 017a8429196619e4c293e69ec8b529c1 in 938ms, sequenceid=700, compaction requested=true 2024-11-19T15:16:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-19T15:16:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-19T15:16:33,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-19T15:16:33,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0920 sec 2024-11-19T15:16:33,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.0970 sec 2024-11-19T15:16:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T15:16:33,814 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:33,815 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T15:16:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-19T15:16:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T15:16:33,817 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T15:16:33,818 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T15:16:33,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T15:16:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T15:16:33,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46557 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T15:16:33,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:33,970 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T15:16:33,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:33,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:33,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:33,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:33,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:33,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193c6016c6803b4963ab088a9219b91a43_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029393056/Put/seqid=0 2024-11-19T15:16:33,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742444_1620 (size=12454) 2024-11-19T15:16:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T15:16:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:34,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. as already flushing 2024-11-19T15:16:34,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029454228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029454232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029454233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029454235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029454344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029454344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029454345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029454345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:34,390 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193c6016c6803b4963ab088a9219b91a43_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193c6016c6803b4963ab088a9219b91a43_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:34,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/59c45ffaea6c42a3ae3e0d275cdcff9b, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:34,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/59c45ffaea6c42a3ae3e0d275cdcff9b is 175, key is test_row_0/A:col10/1732029393056/Put/seqid=0 2024-11-19T15:16:34,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742445_1621 (size=31255) 2024-11-19T15:16:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T15:16:34,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029454555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029454555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029454555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029454556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,795 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=718, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/59c45ffaea6c42a3ae3e0d275cdcff9b 2024-11-19T15:16:34,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/3a621c0993814c2e985595c43eb46322 is 50, key is test_row_0/B:col10/1732029393056/Put/seqid=0 2024-11-19T15:16:34,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742446_1622 (size=12301) 2024-11-19T15:16:34,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029454864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029454864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029454864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029454864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T15:16:35,207 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=718 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/3a621c0993814c2e985595c43eb46322 2024-11-19T15:16:35,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/e1dc9e0db49344bf88e17affb8d71b71 is 50, key is test_row_0/C:col10/1732029393056/Put/seqid=0 2024-11-19T15:16:35,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742447_1623 (size=12301) 2024-11-19T15:16:35,218 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=718 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/e1dc9e0db49344bf88e17affb8d71b71 2024-11-19T15:16:35,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/59c45ffaea6c42a3ae3e0d275cdcff9b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/59c45ffaea6c42a3ae3e0d275cdcff9b 2024-11-19T15:16:35,225 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/59c45ffaea6c42a3ae3e0d275cdcff9b, entries=150, sequenceid=718, filesize=30.5 K 2024-11-19T15:16:35,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/3a621c0993814c2e985595c43eb46322 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3a621c0993814c2e985595c43eb46322 2024-11-19T15:16:35,228 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3a621c0993814c2e985595c43eb46322, entries=150, sequenceid=718, filesize=12.0 K 2024-11-19T15:16:35,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/e1dc9e0db49344bf88e17affb8d71b71 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/e1dc9e0db49344bf88e17affb8d71b71 2024-11-19T15:16:35,232 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/e1dc9e0db49344bf88e17affb8d71b71, entries=150, sequenceid=718, filesize=12.0 K 2024-11-19T15:16:35,233 INFO [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 017a8429196619e4c293e69ec8b529c1 in 1263ms, sequenceid=718, compaction requested=true 2024-11-19T15:16:35,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:35,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:35,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/50570b5b74e7:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-19T15:16:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-19T15:16:35,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-19T15:16:35,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-19T15:16:35,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.4220 sec 2024-11-19T15:16:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:35,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T15:16:35,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:35,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:35,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:35,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:35,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:35,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:35,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119aa603ec987b34bada8637febc74a5b5a_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029395378/Put/seqid=0 2024-11-19T15:16:35,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742448_1624 (size=12454) 2024-11-19T15:16:35,396 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:35,400 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119aa603ec987b34bada8637febc74a5b5a_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119aa603ec987b34bada8637febc74a5b5a_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:35,400 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/29a8818dd0bc4741900faee9b8cade38, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:35,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/29a8818dd0bc4741900faee9b8cade38 is 175, key is test_row_0/A:col10/1732029395378/Put/seqid=0 2024-11-19T15:16:35,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029455397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029455399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:35,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742449_1625 (size=31255) 2024-11-19T15:16:35,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029455401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:35,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029455402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:35,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029455514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:35,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029455514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:35,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029455514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029455514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:35,626 INFO [Thread-2451 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2451. 2024-11-19T15:16:35,626 DEBUG [Thread-2451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:35,626 DEBUG [Thread-2451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:35,626 DEBUG [Thread-2451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:35,626 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:35,626 INFO [Thread-2449 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2449. 2024-11-19T15:16:35,626 DEBUG [Thread-2449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:35,626 DEBUG [Thread-2449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:35,626 DEBUG [Thread-2449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:35,626 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:35,630 INFO [Thread-2457 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2457. 2024-11-19T15:16:35,630 DEBUG [Thread-2457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:35,630 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:35,630 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:35,630 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:35,631 INFO [Thread-2455 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2455. 2024-11-19T15:16:35,631 DEBUG [Thread-2455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:35,631 DEBUG [Thread-2455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:35,631 DEBUG [Thread-2455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:35,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:35,632 INFO [Thread-2453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2453. 2024-11-19T15:16:35,632 DEBUG [Thread-2453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:35,632 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:35,632 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:35,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:35,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:35,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:35,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029455724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029455724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029455724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029455724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:35,806 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=734, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/29a8818dd0bc4741900faee9b8cade38 2024-11-19T15:16:35,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/895d9a5fd62a440298aa074b17f27141 is 50, key is test_row_0/B:col10/1732029395378/Put/seqid=0 2024-11-19T15:16:35,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742450_1626 (size=12301) 2024-11-19T15:16:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T15:16:35,964 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029456034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029456034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029456034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029456034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:36,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=734 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/895d9a5fd62a440298aa074b17f27141 2024-11-19T15:16:36,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/7603c0451c1f45d3ab2f00344028afac is 50, key is test_row_0/C:col10/1732029395378/Put/seqid=0 2024-11-19T15:16:36,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742451_1627 (size=12301) 2024-11-19T15:16:36,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:36,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:36,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53336 deadline: 1732029456544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53310 deadline: 1732029456544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53328 deadline: 1732029456545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:36,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T15:16:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53318 deadline: 1732029456545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 , the old value is region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T15:16:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., hostname=50570b5b74e7,46557,1732029218639, seqNum=5 because the exception is null or not the one we care about 2024-11-19T15:16:36,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=734 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/7603c0451c1f45d3ab2f00344028afac 2024-11-19T15:16:36,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/29a8818dd0bc4741900faee9b8cade38 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/29a8818dd0bc4741900faee9b8cade38 2024-11-19T15:16:36,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/29a8818dd0bc4741900faee9b8cade38, entries=150, sequenceid=734, filesize=30.5 K 2024-11-19T15:16:36,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/895d9a5fd62a440298aa074b17f27141 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/895d9a5fd62a440298aa074b17f27141 2024-11-19T15:16:36,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/895d9a5fd62a440298aa074b17f27141, entries=150, sequenceid=734, filesize=12.0 K 2024-11-19T15:16:36,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/7603c0451c1f45d3ab2f00344028afac as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7603c0451c1f45d3ab2f00344028afac 2024-11-19T15:16:36,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7603c0451c1f45d3ab2f00344028afac, entries=150, sequenceid=734, filesize=12.0 K 2024-11-19T15:16:36,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 017a8429196619e4c293e69ec8b529c1 in 1258ms, sequenceid=734, compaction requested=true 2024-11-19T15:16:36,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:36,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T15:16:36,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:36,637 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-19T15:16:36,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T15:16:36,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:36,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 017a8429196619e4c293e69ec8b529c1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T15:16:36,637 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-19T15:16:36,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:36,639 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 188008 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-19T15:16:36,639 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74284 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-19T15:16:36,639 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/A is initiating minor compaction (all files) 2024-11-19T15:16:36,639 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/B is initiating minor compaction (all files) 2024-11-19T15:16:36,639 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/B in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:36,639 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/A in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:36,639 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d39a0a949a9e4eb5a80fb2e864eee15f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/327078dd0b80462daf8126086936a30d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/88925e91ed044934988fdbdd8fe298b8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a09fbf3da695414ead87f45f7723f7fe, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3a621c0993814c2e985595c43eb46322, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/895d9a5fd62a440298aa074b17f27141] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=72.5 K 2024-11-19T15:16:36,639 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/7ea217b4194c4d659e4b63bc0895cb48, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/b3dc4048e7824a84a29b2723b10797f3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/e45a357bfdb74cfaa02275d9f8ae2bb7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f247d597d06a49a2b57fcbe8c7c10e02, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/59c45ffaea6c42a3ae3e0d275cdcff9b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/29a8818dd0bc4741900faee9b8cade38] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=183.6 K 2024-11-19T15:16:36,639 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:36,639 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. files: [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/7ea217b4194c4d659e4b63bc0895cb48, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/b3dc4048e7824a84a29b2723b10797f3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/e45a357bfdb74cfaa02275d9f8ae2bb7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f247d597d06a49a2b57fcbe8c7c10e02, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/59c45ffaea6c42a3ae3e0d275cdcff9b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/29a8818dd0bc4741900faee9b8cade38] 2024-11-19T15:16:36,639 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting d39a0a949a9e4eb5a80fb2e864eee15f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732029388367 2024-11-19T15:16:36,639 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ea217b4194c4d659e4b63bc0895cb48, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732029388367 2024-11-19T15:16:36,639 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 327078dd0b80462daf8126086936a30d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1732029389519 2024-11-19T15:16:36,639 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3dc4048e7824a84a29b2723b10797f3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1732029389519 2024-11-19T15:16:36,640 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting e45a357bfdb74cfaa02275d9f8ae2bb7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=686, earliestPutTs=1732029390698 2024-11-19T15:16:36,640 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 88925e91ed044934988fdbdd8fe298b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=686, earliestPutTs=1732029390698 2024-11-19T15:16:36,640 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting f247d597d06a49a2b57fcbe8c7c10e02, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=700, earliestPutTs=1732029391858 2024-11-19T15:16:36,640 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting a09fbf3da695414ead87f45f7723f7fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=700, earliestPutTs=1732029391858 2024-11-19T15:16:36,640 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59c45ffaea6c42a3ae3e0d275cdcff9b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=718, earliestPutTs=1732029393056 2024-11-19T15:16:36,640 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a621c0993814c2e985595c43eb46322, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=718, earliestPutTs=1732029393056 2024-11-19T15:16:36,640 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 895d9a5fd62a440298aa074b17f27141, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=734, earliestPutTs=1732029394230 2024-11-19T15:16:36,640 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29a8818dd0bc4741900faee9b8cade38, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=734, earliestPutTs=1732029394230 2024-11-19T15:16:36,647 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:36,648 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119bcbaaeb9b2484720aadbdb44f82c6882_017a8429196619e4c293e69ec8b529c1 store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:36,648 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#B#compaction#530 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:36,649 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/3f5149de5575479381f82ec49ae1ceac is 50, key is test_row_0/B:col10/1732029395378/Put/seqid=0 2024-11-19T15:16:36,652 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119bcbaaeb9b2484720aadbdb44f82c6882_017a8429196619e4c293e69ec8b529c1, store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:36,652 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bcbaaeb9b2484720aadbdb44f82c6882_017a8429196619e4c293e69ec8b529c1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:36,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742452_1628 (size=12983) 2024-11-19T15:16:36,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742453_1629 (size=4469) 2024-11-19T15:16:36,980 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T15:16:37,056 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/3f5149de5575479381f82ec49ae1ceac as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3f5149de5575479381f82ec49ae1ceac 2024-11-19T15:16:37,056 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#A#compaction#531 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:37,057 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/6e259b64a93047f7a4279821a15c4c90 is 175, key is test_row_0/A:col10/1732029395378/Put/seqid=0 2024-11-19T15:16:37,059 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/B of 017a8429196619e4c293e69ec8b529c1 into 3f5149de5575479381f82ec49ae1ceac(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:37,059 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:37,059 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/B, priority=10, startTime=1732029396637; duration=0sec 2024-11-19T15:16:37,059 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T15:16:37,059 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:B 2024-11-19T15:16:37,060 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-19T15:16:37,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742454_1630 (size=31937) 2024-11-19T15:16:37,061 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74284 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-19T15:16:37,061 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1541): 017a8429196619e4c293e69ec8b529c1/C is initiating minor compaction (all files) 2024-11-19T15:16:37,061 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 017a8429196619e4c293e69ec8b529c1/C in TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:37,061 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/448d24d43b1a4085a93a9bf9a57c37a4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/17e94e38e35740299ec981e7812ccc00, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7e83730b046147b38ba7607323a93644, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/16ed5e088c4b4492b7ac5f83a038bebd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/e1dc9e0db49344bf88e17affb8d71b71, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7603c0451c1f45d3ab2f00344028afac] into tmpdir=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp, totalSize=72.5 K 2024-11-19T15:16:37,061 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 448d24d43b1a4085a93a9bf9a57c37a4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732029388367 2024-11-19T15:16:37,061 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 17e94e38e35740299ec981e7812ccc00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1732029389519 2024-11-19T15:16:37,061 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e83730b046147b38ba7607323a93644, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=686, earliestPutTs=1732029390698 2024-11-19T15:16:37,061 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 16ed5e088c4b4492b7ac5f83a038bebd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=700, earliestPutTs=1732029391858 2024-11-19T15:16:37,062 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting e1dc9e0db49344bf88e17affb8d71b71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=718, earliestPutTs=1732029393056 2024-11-19T15:16:37,062 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] compactions.Compactor(225): Compacting 7603c0451c1f45d3ab2f00344028afac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=734, earliestPutTs=1732029394230 2024-11-19T15:16:37,069 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 017a8429196619e4c293e69ec8b529c1#C#compaction#532 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T15:16:37,070 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/a8f508553cdc4245b0df1cc9d9e92741 is 50, key is test_row_0/C:col10/1732029395378/Put/seqid=0 2024-11-19T15:16:37,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742455_1631 (size=12983) 2024-11-19T15:16:37,463 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/6e259b64a93047f7a4279821a15c4c90 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/6e259b64a93047f7a4279821a15c4c90 2024-11-19T15:16:37,466 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/A of 017a8429196619e4c293e69ec8b529c1 into 6e259b64a93047f7a4279821a15c4c90(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:37,466 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:37,466 INFO [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/A, priority=10, startTime=1732029396637; duration=0sec 2024-11-19T15:16:37,467 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:37,467 DEBUG [RS:0;50570b5b74e7:46557-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:A 2024-11-19T15:16:37,475 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/a8f508553cdc4245b0df1cc9d9e92741 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a8f508553cdc4245b0df1cc9d9e92741 2024-11-19T15:16:37,478 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 017a8429196619e4c293e69ec8b529c1/C of 017a8429196619e4c293e69ec8b529c1 into a8f508553cdc4245b0df1cc9d9e92741(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T15:16:37,478 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:37,478 INFO [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., storeName=017a8429196619e4c293e69ec8b529c1/C, priority=10, startTime=1732029396637; duration=0sec 2024-11-19T15:16:37,478 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T15:16:37,478 DEBUG [RS:0;50570b5b74e7:46557-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 017a8429196619e4c293e69ec8b529c1:C 2024-11-19T15:16:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46557 {}] regionserver.HRegion(8855): Flush requested on 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:37,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T15:16:37,557 INFO [Thread-2446 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2446. 2024-11-19T15:16:37,557 INFO [Thread-2444 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2444. 2024-11-19T15:16:37,557 INFO [Thread-2438 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2438. 2024-11-19T15:16:37,557 DEBUG [Thread-2446 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:37,557 DEBUG [Thread-2444 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:37,557 DEBUG [Thread-2438 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:37,557 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:37,557 DEBUG [Thread-2444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:37,557 DEBUG [Thread-2438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:37,557 DEBUG [Thread-2444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:37,557 DEBUG [Thread-2438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:37,557 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:37,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:37,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:37,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:37,559 INFO [Thread-2442 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2442. 2024-11-19T15:16:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:37,559 DEBUG [Thread-2442 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:37,559 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:37,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:37,559 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:37,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:37,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bdc562bd21d145cda7bc7639a3f63377_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029397555/Put/seqid=0 2024-11-19T15:16:37,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742456_1632 (size=12454) 2024-11-19T15:16:37,825 INFO [Thread-2440 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2440. 2024-11-19T15:16:37,825 DEBUG [Thread-2440 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T15:16:37,825 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:37,825 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:37,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:37,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T15:16:37,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 140 2024-11-19T15:16:37,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-19T15:16:37,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 210 2024-11-19T15:16:37,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 221 2024-11-19T15:16:37,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-19T15:16:37,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T15:16:37,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2845 2024-11-19T15:16:37,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2753 2024-11-19T15:16:37,826 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2872 2024-11-19T15:16:37,826 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2849 2024-11-19T15:16:37,826 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2774 2024-11-19T15:16:37,826 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T15:16:37,826 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T15:16:37,826 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T15:16:37,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:37,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:37,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:37,827 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T15:16:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T15:16:37,829 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029397829"}]},"ts":"1732029397829"} 2024-11-19T15:16:37,830 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T15:16:37,831 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T15:16:37,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T15:16:37,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, UNASSIGN}] 2024-11-19T15:16:37,833 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, UNASSIGN 2024-11-19T15:16:37,834 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=017a8429196619e4c293e69ec8b529c1, regionState=CLOSING, regionLocation=50570b5b74e7,46557,1732029218639 2024-11-19T15:16:37,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, UNASSIGN because future has completed 2024-11-19T15:16:37,835 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T15:16:37,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639}] 2024-11-19T15:16:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T15:16:37,967 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:37,970 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bdc562bd21d145cda7bc7639a3f63377_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bdc562bd21d145cda7bc7639a3f63377_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:37,971 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/bdf43687ef4f481196d70f22d74ffe2d, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:37,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/bdf43687ef4f481196d70f22d74ffe2d is 175, key is test_row_0/A:col10/1732029397555/Put/seqid=0 2024-11-19T15:16:37,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742457_1633 (size=31255) 2024-11-19T15:16:37,987 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:37,987 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T15:16:37,987 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing 017a8429196619e4c293e69ec8b529c1, disabling compactions & flushes 2024-11-19T15:16:37,987 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T15:16:38,374 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=757, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/bdf43687ef4f481196d70f22d74ffe2d 2024-11-19T15:16:38,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/a98a550524eb4881b4b539457ad44e5d is 50, key is test_row_0/B:col10/1732029397555/Put/seqid=0 2024-11-19T15:16:38,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742458_1634 (size=12301) 2024-11-19T15:16:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T15:16:38,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=757 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/a98a550524eb4881b4b539457ad44e5d 2024-11-19T15:16:38,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/c4ff3323e6034e5abb70ba6afb74895b is 50, key is test_row_0/C:col10/1732029397555/Put/seqid=0 2024-11-19T15:16:38,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742459_1635 (size=12301) 2024-11-19T15:16:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T15:16:39,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=757 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/c4ff3323e6034e5abb70ba6afb74895b 2024-11-19T15:16:39,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/bdf43687ef4f481196d70f22d74ffe2d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/bdf43687ef4f481196d70f22d74ffe2d 2024-11-19T15:16:39,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/bdf43687ef4f481196d70f22d74ffe2d, entries=150, sequenceid=757, filesize=30.5 K 2024-11-19T15:16:39,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/a98a550524eb4881b4b539457ad44e5d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a98a550524eb4881b4b539457ad44e5d 2024-11-19T15:16:39,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a98a550524eb4881b4b539457ad44e5d, entries=150, sequenceid=757, filesize=12.0 K 2024-11-19T15:16:39,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/c4ff3323e6034e5abb70ba6afb74895b as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/c4ff3323e6034e5abb70ba6afb74895b 2024-11-19T15:16:39,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/c4ff3323e6034e5abb70ba6afb74895b, entries=150, sequenceid=757, filesize=12.0 K 2024-11-19T15:16:39,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for 017a8429196619e4c293e69ec8b529c1 in 1646ms, sequenceid=757, compaction requested=false 2024-11-19T15:16:39,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 017a8429196619e4c293e69ec8b529c1: 2024-11-19T15:16:39,203 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:39,203 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:39,203 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. after waiting 0 ms 2024-11-19T15:16:39,203 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:39,203 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing 017a8429196619e4c293e69ec8b529c1 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-19T15:16:39,203 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=A 2024-11-19T15:16:39,203 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:39,203 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=B 2024-11-19T15:16:39,203 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:39,203 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 017a8429196619e4c293e69ec8b529c1, store=C 2024-11-19T15:16:39,203 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T15:16:39,207 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b7058e9ce30945ba84590e29632ca902_017a8429196619e4c293e69ec8b529c1 is 50, key is test_row_0/A:col10/1732029397824/Put/seqid=0 2024-11-19T15:16:39,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742460_1636 (size=7374) 2024-11-19T15:16:39,610 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T15:16:39,613 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b7058e9ce30945ba84590e29632ca902_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b7058e9ce30945ba84590e29632ca902_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:39,614 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/0946cbaad98342b3912d1822c9dff179, store: [table=TestAcidGuarantees family=A region=017a8429196619e4c293e69ec8b529c1] 2024-11-19T15:16:39,614 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/0946cbaad98342b3912d1822c9dff179 is 175, key is test_row_0/A:col10/1732029397824/Put/seqid=0 2024-11-19T15:16:39,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742461_1637 (size=13865) 2024-11-19T15:16:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T15:16:40,018 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=761, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/0946cbaad98342b3912d1822c9dff179 2024-11-19T15:16:40,023 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/d88f00bd7b984a1f8b249aa56f040bad is 50, key is test_row_0/B:col10/1732029397824/Put/seqid=0 2024-11-19T15:16:40,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742462_1638 (size=7415) 2024-11-19T15:16:40,426 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=761 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/d88f00bd7b984a1f8b249aa56f040bad 2024-11-19T15:16:40,431 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/faa391315aae48558b044d40235045f7 is 50, key is test_row_0/C:col10/1732029397824/Put/seqid=0 2024-11-19T15:16:40,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742463_1639 (size=7415) 2024-11-19T15:16:40,834 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=761 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/faa391315aae48558b044d40235045f7 2024-11-19T15:16:40,838 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/A/0946cbaad98342b3912d1822c9dff179 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/0946cbaad98342b3912d1822c9dff179 2024-11-19T15:16:40,840 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/0946cbaad98342b3912d1822c9dff179, entries=50, sequenceid=761, filesize=13.5 K 2024-11-19T15:16:40,841 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/B/d88f00bd7b984a1f8b249aa56f040bad as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d88f00bd7b984a1f8b249aa56f040bad 2024-11-19T15:16:40,843 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d88f00bd7b984a1f8b249aa56f040bad, entries=50, sequenceid=761, filesize=7.2 K 2024-11-19T15:16:40,844 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/.tmp/C/faa391315aae48558b044d40235045f7 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/faa391315aae48558b044d40235045f7 2024-11-19T15:16:40,846 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/faa391315aae48558b044d40235045f7, entries=50, sequenceid=761, filesize=7.2 K 2024-11-19T15:16:40,847 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 017a8429196619e4c293e69ec8b529c1 in 1644ms, sequenceid=761, compaction requested=true 2024-11-19T15:16:40,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/2a619cf2ff0f4311aea60a19aaca3faf, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/624679725a3e4509aaabb49d4fae5136, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f66c8f995eb2463f9e245f032ee63994, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/4330593e210e4417883f672920e8503e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/db6ac21e756644fbacbfbc373a9c8015, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/517924c75f1a4d339522e36431f186bd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/72a575a2e3f84f37b0a29d7250d6c643, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/22c2aaea4100492fac7af7cb7df89833, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/87633ad6378c4d3f873c0134c631c7be, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/0568911062dd40ff8e75e7c000e0fd02, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d5e7359a99fe48098d96324c708190af, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d1b4cc88126440dd8802ec6236ea4655, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/09837b0e7265479e815fbe7a6a496ce0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/7ea217b4194c4d659e4b63bc0895cb48, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f167e5e24c9042479b451ea1826dbf9d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/b3dc4048e7824a84a29b2723b10797f3, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/e45a357bfdb74cfaa02275d9f8ae2bb7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f247d597d06a49a2b57fcbe8c7c10e02, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/59c45ffaea6c42a3ae3e0d275cdcff9b, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/29a8818dd0bc4741900faee9b8cade38] to archive 2024-11-19T15:16:40,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:16:40,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/2a619cf2ff0f4311aea60a19aaca3faf to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/2a619cf2ff0f4311aea60a19aaca3faf 2024-11-19T15:16:40,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/624679725a3e4509aaabb49d4fae5136 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/624679725a3e4509aaabb49d4fae5136 2024-11-19T15:16:40,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f66c8f995eb2463f9e245f032ee63994 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f66c8f995eb2463f9e245f032ee63994 2024-11-19T15:16:40,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/4330593e210e4417883f672920e8503e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/4330593e210e4417883f672920e8503e 2024-11-19T15:16:40,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/db6ac21e756644fbacbfbc373a9c8015 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/db6ac21e756644fbacbfbc373a9c8015 2024-11-19T15:16:40,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/517924c75f1a4d339522e36431f186bd to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/517924c75f1a4d339522e36431f186bd 2024-11-19T15:16:40,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/72a575a2e3f84f37b0a29d7250d6c643 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/72a575a2e3f84f37b0a29d7250d6c643 2024-11-19T15:16:40,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/22c2aaea4100492fac7af7cb7df89833 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/22c2aaea4100492fac7af7cb7df89833 2024-11-19T15:16:40,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/87633ad6378c4d3f873c0134c631c7be to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/87633ad6378c4d3f873c0134c631c7be 2024-11-19T15:16:40,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/0568911062dd40ff8e75e7c000e0fd02 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/0568911062dd40ff8e75e7c000e0fd02 2024-11-19T15:16:40,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d5e7359a99fe48098d96324c708190af to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d5e7359a99fe48098d96324c708190af 2024-11-19T15:16:40,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d1b4cc88126440dd8802ec6236ea4655 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/d1b4cc88126440dd8802ec6236ea4655 2024-11-19T15:16:40,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/09837b0e7265479e815fbe7a6a496ce0 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/09837b0e7265479e815fbe7a6a496ce0 2024-11-19T15:16:40,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/7ea217b4194c4d659e4b63bc0895cb48 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/7ea217b4194c4d659e4b63bc0895cb48 2024-11-19T15:16:40,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f167e5e24c9042479b451ea1826dbf9d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f167e5e24c9042479b451ea1826dbf9d 2024-11-19T15:16:40,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/b3dc4048e7824a84a29b2723b10797f3 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/b3dc4048e7824a84a29b2723b10797f3 2024-11-19T15:16:40,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/e45a357bfdb74cfaa02275d9f8ae2bb7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/e45a357bfdb74cfaa02275d9f8ae2bb7 2024-11-19T15:16:40,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f247d597d06a49a2b57fcbe8c7c10e02 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/f247d597d06a49a2b57fcbe8c7c10e02 2024-11-19T15:16:40,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/59c45ffaea6c42a3ae3e0d275cdcff9b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/59c45ffaea6c42a3ae3e0d275cdcff9b 2024-11-19T15:16:40,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/29a8818dd0bc4741900faee9b8cade38 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/29a8818dd0bc4741900faee9b8cade38 2024-11-19T15:16:40,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/9f7f9e1cc6084eb2898999ccbec86467, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/1a5f68f1d5564eee9eb476f888df14b0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/feacff36651d4c9c9c9d82e98d23f737, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/6c6ca60cd4ec4e4da6736614d72ec675, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/dd75489a178f4dd3b4eb0de69e86d23a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/7ca6f1a63090450c8cf397072dafb30e, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/5df2fc8d1b9c47c3a0efce5fed52a262, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/629796579b6c426285c76eae3ca3f464, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a1fc4e9240804d5f905f8b5523044689, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d73c60b51ad64fa5b42a9b49e7b28981, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/61c1d1d0572f42c1914b5dce6479a09a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3cabd70e41b9443e92322a94a0e3722c, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3b49389e45be4b068249841e94ff1bbf, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d39a0a949a9e4eb5a80fb2e864eee15f, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/f1f83df1b6114a27ac6e2a884603c78a, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/327078dd0b80462daf8126086936a30d, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/88925e91ed044934988fdbdd8fe298b8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a09fbf3da695414ead87f45f7723f7fe, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3a621c0993814c2e985595c43eb46322, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/895d9a5fd62a440298aa074b17f27141] to archive 2024-11-19T15:16:40,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:16:40,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/9f7f9e1cc6084eb2898999ccbec86467 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/9f7f9e1cc6084eb2898999ccbec86467 2024-11-19T15:16:40,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/1a5f68f1d5564eee9eb476f888df14b0 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/1a5f68f1d5564eee9eb476f888df14b0 2024-11-19T15:16:40,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/feacff36651d4c9c9c9d82e98d23f737 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/feacff36651d4c9c9c9d82e98d23f737 2024-11-19T15:16:40,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/6c6ca60cd4ec4e4da6736614d72ec675 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/6c6ca60cd4ec4e4da6736614d72ec675 2024-11-19T15:16:40,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/dd75489a178f4dd3b4eb0de69e86d23a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/dd75489a178f4dd3b4eb0de69e86d23a 2024-11-19T15:16:40,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/7ca6f1a63090450c8cf397072dafb30e to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/7ca6f1a63090450c8cf397072dafb30e 2024-11-19T15:16:40,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/5df2fc8d1b9c47c3a0efce5fed52a262 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/5df2fc8d1b9c47c3a0efce5fed52a262 2024-11-19T15:16:40,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/629796579b6c426285c76eae3ca3f464 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/629796579b6c426285c76eae3ca3f464 2024-11-19T15:16:40,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a1fc4e9240804d5f905f8b5523044689 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a1fc4e9240804d5f905f8b5523044689 2024-11-19T15:16:40,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d73c60b51ad64fa5b42a9b49e7b28981 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d73c60b51ad64fa5b42a9b49e7b28981 2024-11-19T15:16:40,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/61c1d1d0572f42c1914b5dce6479a09a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/61c1d1d0572f42c1914b5dce6479a09a 2024-11-19T15:16:40,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3cabd70e41b9443e92322a94a0e3722c to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3cabd70e41b9443e92322a94a0e3722c 2024-11-19T15:16:40,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3b49389e45be4b068249841e94ff1bbf to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3b49389e45be4b068249841e94ff1bbf 2024-11-19T15:16:40,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d39a0a949a9e4eb5a80fb2e864eee15f to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d39a0a949a9e4eb5a80fb2e864eee15f 2024-11-19T15:16:40,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/f1f83df1b6114a27ac6e2a884603c78a to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/f1f83df1b6114a27ac6e2a884603c78a 2024-11-19T15:16:40,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/327078dd0b80462daf8126086936a30d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/327078dd0b80462daf8126086936a30d 2024-11-19T15:16:40,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/88925e91ed044934988fdbdd8fe298b8 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/88925e91ed044934988fdbdd8fe298b8 2024-11-19T15:16:40,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a09fbf3da695414ead87f45f7723f7fe to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a09fbf3da695414ead87f45f7723f7fe 2024-11-19T15:16:40,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3a621c0993814c2e985595c43eb46322 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3a621c0993814c2e985595c43eb46322 2024-11-19T15:16:40,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/895d9a5fd62a440298aa074b17f27141 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/895d9a5fd62a440298aa074b17f27141 2024-11-19T15:16:40,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7de4254f979c416f80b36e8f230eb064, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/4fcfe0996bbd48fc98827e038c1d7da0, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/10ad70c2b8e24234b99f9ce70968b883, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/eeedab8348c5411ca437e60348cc4d20, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/17621af7af784cf8a087af60e0091eb2, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a25035af17fb42db8f2314dde455d021, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/47bbb83830894c6fbbeada2f7049bfac, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/d330aa1297984538a79291682b4ca624, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a47269a1de27436ab71ca4b962726fea, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/d431b3db20894a468191a0a675c822e8, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/9831c5c5d105492d801064af1b5a6ab7, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/f8c8b873c357474e80cc56ececc5c0d9, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/6783cf2e0c604e48a2db1fce7c558d01, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/448d24d43b1a4085a93a9bf9a57c37a4, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/f5680694c07c4388ae1a4d969f982eb5, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/17e94e38e35740299ec981e7812ccc00, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7e83730b046147b38ba7607323a93644, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/16ed5e088c4b4492b7ac5f83a038bebd, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/e1dc9e0db49344bf88e17affb8d71b71, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7603c0451c1f45d3ab2f00344028afac] to archive 2024-11-19T15:16:40,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T15:16:40,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7de4254f979c416f80b36e8f230eb064 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7de4254f979c416f80b36e8f230eb064 2024-11-19T15:16:40,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/4fcfe0996bbd48fc98827e038c1d7da0 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/4fcfe0996bbd48fc98827e038c1d7da0 2024-11-19T15:16:40,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/10ad70c2b8e24234b99f9ce70968b883 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/10ad70c2b8e24234b99f9ce70968b883 2024-11-19T15:16:40,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/eeedab8348c5411ca437e60348cc4d20 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/eeedab8348c5411ca437e60348cc4d20 2024-11-19T15:16:40,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/17621af7af784cf8a087af60e0091eb2 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/17621af7af784cf8a087af60e0091eb2 2024-11-19T15:16:40,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a25035af17fb42db8f2314dde455d021 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a25035af17fb42db8f2314dde455d021 2024-11-19T15:16:40,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/47bbb83830894c6fbbeada2f7049bfac to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/47bbb83830894c6fbbeada2f7049bfac 2024-11-19T15:16:40,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/d330aa1297984538a79291682b4ca624 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/d330aa1297984538a79291682b4ca624 2024-11-19T15:16:40,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a47269a1de27436ab71ca4b962726fea to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a47269a1de27436ab71ca4b962726fea 2024-11-19T15:16:40,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/d431b3db20894a468191a0a675c822e8 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/d431b3db20894a468191a0a675c822e8 2024-11-19T15:16:40,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/9831c5c5d105492d801064af1b5a6ab7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/9831c5c5d105492d801064af1b5a6ab7 2024-11-19T15:16:40,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/f8c8b873c357474e80cc56ececc5c0d9 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/f8c8b873c357474e80cc56ececc5c0d9 2024-11-19T15:16:40,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/6783cf2e0c604e48a2db1fce7c558d01 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/6783cf2e0c604e48a2db1fce7c558d01 2024-11-19T15:16:40,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/448d24d43b1a4085a93a9bf9a57c37a4 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/448d24d43b1a4085a93a9bf9a57c37a4 2024-11-19T15:16:40,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/f5680694c07c4388ae1a4d969f982eb5 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/f5680694c07c4388ae1a4d969f982eb5 2024-11-19T15:16:40,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/17e94e38e35740299ec981e7812ccc00 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/17e94e38e35740299ec981e7812ccc00 2024-11-19T15:16:40,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7e83730b046147b38ba7607323a93644 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7e83730b046147b38ba7607323a93644 2024-11-19T15:16:40,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/16ed5e088c4b4492b7ac5f83a038bebd to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/16ed5e088c4b4492b7ac5f83a038bebd 2024-11-19T15:16:40,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/e1dc9e0db49344bf88e17affb8d71b71 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/e1dc9e0db49344bf88e17affb8d71b71 2024-11-19T15:16:40,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7603c0451c1f45d3ab2f00344028afac to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/7603c0451c1f45d3ab2f00344028afac 2024-11-19T15:16:40,901 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/recovered.edits/764.seqid, newMaxSeqId=764, maxSeqId=4 2024-11-19T15:16:40,901 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. 2024-11-19T15:16:40,901 DEBUG [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for 017a8429196619e4c293e69ec8b529c1: Waiting for close lock at 1732029397987Running coprocessor pre-close hooks at 1732029397987Disabling compacts and flushes for region at 1732029397987Disabling writes for close at 1732029399203 (+1216 ms)Obtaining lock to block concurrent updates at 1732029399203Preparing flush snapshotting stores in 017a8429196619e4c293e69ec8b529c1 at 1732029399203Finished memstore snapshotting TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1732029399203Flushing stores of TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1. at 1732029399203Flushing 017a8429196619e4c293e69ec8b529c1/A: creating writer at 1732029399204 (+1 ms)Flushing mob file 017a8429196619e4c293e69ec8b529c1/A: appending metadata at 1732029399207 (+3 ms)Flushing mob file 017a8429196619e4c293e69ec8b529c1/A: closing flushed file at 1732029399207Flushing 017a8429196619e4c293e69ec8b529c1/A: appending metadata at 1732029399614 (+407 ms)Flushing 017a8429196619e4c293e69ec8b529c1/A: closing flushed file at 1732029399614Flushing 017a8429196619e4c293e69ec8b529c1/B: creating writer at 1732029400020 (+406 ms)Flushing 017a8429196619e4c293e69ec8b529c1/B: appending metadata at 1732029400022 (+2 ms)Flushing 017a8429196619e4c293e69ec8b529c1/B: closing flushed file at 1732029400022Flushing 017a8429196619e4c293e69ec8b529c1/C: creating writer at 1732029400429 (+407 ms)Flushing 017a8429196619e4c293e69ec8b529c1/C: appending metadata at 1732029400431 (+2 ms)Flushing 017a8429196619e4c293e69ec8b529c1/C: closing flushed file at 1732029400431Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@276fdc58: reopening flushed file at 1732029400837 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71b1fb1e: reopening flushed file at 1732029400840 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@604f4e6: reopening flushed file at 1732029400844 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 017a8429196619e4c293e69ec8b529c1 in 1644ms, sequenceid=761, compaction requested=true at 1732029400847 (+3 ms)Writing region close event to WAL at 1732029400898 (+51 ms)Running coprocessor post-close hooks at 1732029400901 (+3 ms)Closed at 1732029400901 2024-11-19T15:16:40,903 INFO [RS_CLOSE_REGION-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed 017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:40,903 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=017a8429196619e4c293e69ec8b529c1, regionState=CLOSED 2024-11-19T15:16:40,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 because future has completed 2024-11-19T15:16:40,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-19T15:16:40,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure 017a8429196619e4c293e69ec8b529c1, server=50570b5b74e7,46557,1732029218639 in 3.0700 sec 2024-11-19T15:16:40,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-19T15:16:40,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=017a8429196619e4c293e69ec8b529c1, UNASSIGN in 3.0740 sec 2024-11-19T15:16:40,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-19T15:16:40,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.0770 sec 2024-11-19T15:16:40,910 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732029400910"}]},"ts":"1732029400910"} 2024-11-19T15:16:40,911 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T15:16:40,911 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T15:16:40,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.0840 sec 2024-11-19T15:16:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T15:16:41,994 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:41,994 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T15:16:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:41,996 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-19T15:16:41,997 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:41,998 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,000 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C, FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/recovered.edits] 2024-11-19T15:16:42,002 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/0946cbaad98342b3912d1822c9dff179 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/0946cbaad98342b3912d1822c9dff179 2024-11-19T15:16:42,002 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/6e259b64a93047f7a4279821a15c4c90 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/6e259b64a93047f7a4279821a15c4c90 2024-11-19T15:16:42,003 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/bdf43687ef4f481196d70f22d74ffe2d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/A/bdf43687ef4f481196d70f22d74ffe2d 2024-11-19T15:16:42,005 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3f5149de5575479381f82ec49ae1ceac to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/3f5149de5575479381f82ec49ae1ceac 2024-11-19T15:16:42,005 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a98a550524eb4881b4b539457ad44e5d to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/a98a550524eb4881b4b539457ad44e5d 2024-11-19T15:16:42,006 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d88f00bd7b984a1f8b249aa56f040bad to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/B/d88f00bd7b984a1f8b249aa56f040bad 2024-11-19T15:16:42,008 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a8f508553cdc4245b0df1cc9d9e92741 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/a8f508553cdc4245b0df1cc9d9e92741 2024-11-19T15:16:42,008 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/c4ff3323e6034e5abb70ba6afb74895b to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/c4ff3323e6034e5abb70ba6afb74895b 2024-11-19T15:16:42,009 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/faa391315aae48558b044d40235045f7 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/C/faa391315aae48558b044d40235045f7 2024-11-19T15:16:42,011 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/recovered.edits/764.seqid to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1/recovered.edits/764.seqid 2024-11-19T15:16:42,011 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/default/TestAcidGuarantees/017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,011 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T15:16:42,012 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T15:16:42,012 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T15:16:42,014 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193149049707e242ac825d28b29e56a3b4_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193149049707e242ac825d28b29e56a3b4_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,015 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193c15873f88154f49933110044930e606_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193c15873f88154f49933110044930e606_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,016 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193c6016c6803b4963ab088a9219b91a43_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193c6016c6803b4963ab088a9219b91a43_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,017 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196b0037bcddc8407ca70e5716cb14a64a_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196b0037bcddc8407ca70e5716cb14a64a_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,017 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196ccfd464419a4fda9d10726e96cfff25_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196ccfd464419a4fda9d10726e96cfff25_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,018 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196ee00c7b26284e849a675616216414f2_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196ee00c7b26284e849a675616216414f2_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,019 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111983872b16796045388b54d21c51a833ac_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111983872b16796045388b54d21c51a833ac_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,020 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198f8aced484a04e6d96a038f9d204325d_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198f8aced484a04e6d96a038f9d204325d_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,021 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111994750fff398f4cde86fe840eeb5cd9e8_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111994750fff398f4cde86fe840eeb5cd9e8_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,022 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a706a85f596b4c318e2c6d26ec4763a1_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a706a85f596b4c318e2c6d26ec4763a1_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,023 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a8339d06df984eae9edce9982307a9a4_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a8339d06df984eae9edce9982307a9a4_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,023 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119aa603ec987b34bada8637febc74a5b5a_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119aa603ec987b34bada8637febc74a5b5a_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,024 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac56bcf6ba824f70a1d1897b474f3bbf_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ac56bcf6ba824f70a1d1897b474f3bbf_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,025 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119adcf1a55b6c648ca97e01a2306b6a42f_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119adcf1a55b6c648ca97e01a2306b6a42f_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,026 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b7058e9ce30945ba84590e29632ca902_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b7058e9ce30945ba84590e29632ca902_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,026 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b94208ba8a1145868c71715279cc3610_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b94208ba8a1145868c71715279cc3610_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,027 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ba404de488f5407bb515a4c4de8d811a_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ba404de488f5407bb515a4c4de8d811a_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,028 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bdc562bd21d145cda7bc7639a3f63377_017a8429196619e4c293e69ec8b529c1 to hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bdc562bd21d145cda7bc7639a3f63377_017a8429196619e4c293e69ec8b529c1 2024-11-19T15:16:42,028 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T15:16:42,030 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:42,032 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T15:16:42,033 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T15:16:42,034 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:42,034 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T15:16:42,034 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732029402034"}]},"ts":"9223372036854775807"} 2024-11-19T15:16:42,036 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T15:16:42,036 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 017a8429196619e4c293e69ec8b529c1, NAME => 'TestAcidGuarantees,,1732029372522.017a8429196619e4c293e69ec8b529c1.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T15:16:42,036 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T15:16:42,036 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732029402036"}]},"ts":"9223372036854775807"} 2024-11-19T15:16:42,037 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T15:16:42,038 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T15:16:42,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-19T15:16:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-19T15:16:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T15:16:42,105 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T15:16:42,114 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 248), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=419 (was 439), ProcessCount=11 (was 11), AvailableMemoryMB=7267 (was 7328) 2024-11-19T15:16:42,114 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T15:16:42,114 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T15:16:42,114 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T15:16:42,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:42,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:42,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T15:16:42,115 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T15:16:42,115 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1143773905, stopped=false 2024-11-19T15:16:42,115 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=50570b5b74e7,39501,1732029217890 2024-11-19T15:16:42,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T15:16:42,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T15:16:42,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:16:42,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:16:42,118 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T15:16:42,119 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T15:16:42,119 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T15:16:42,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:42,119 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T15:16:42,119 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '50570b5b74e7,46557,1732029218639' ***** 2024-11-19T15:16:42,119 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T15:16:42,119 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T15:16:42,120 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T15:16:42,120 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T15:16:42,120 INFO [RS:0;50570b5b74e7:46557 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T15:16:42,120 INFO [RS:0;50570b5b74e7:46557 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T15:16:42,120 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(959): stopping server 50570b5b74e7,46557,1732029218639 2024-11-19T15:16:42,120 INFO [RS:0;50570b5b74e7:46557 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T15:16:42,120 INFO [RS:0;50570b5b74e7:46557 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;50570b5b74e7:46557. 2024-11-19T15:16:42,120 DEBUG [RS:0;50570b5b74e7:46557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T15:16:42,120 DEBUG [RS:0;50570b5b74e7:46557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:42,121 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T15:16:42,121 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T15:16:42,121 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T15:16:42,121 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T15:16:42,121 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T15:16:42,121 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T15:16:42,122 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T15:16:42,122 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T15:16:42,122 INFO [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T15:16:42,122 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T15:16:42,122 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T15:16:42,122 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T15:16:42,122 INFO [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-19T15:16:42,144 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/.tmp/info/468f9bec671146938326ccc0f3328000 is 95, key is TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480./info:/1732029249112/DeleteFamily/seqid=0 2024-11-19T15:16:42,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742464_1640 (size=6514) 2024-11-19T15:16:42,173 INFO [regionserver/50570b5b74e7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T15:16:42,245 INFO [regionserver/50570b5b74e7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T15:16:42,245 INFO [regionserver/50570b5b74e7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T15:16:42,322 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T15:16:42,522 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T15:16:42,548 INFO [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/.tmp/info/468f9bec671146938326ccc0f3328000 2024-11-19T15:16:42,566 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/.tmp/ns/bb5f9e01185e4dbdaa78c0cbff75fd80 is 93, key is TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480./ns:/1732029249045/DeleteFamily/seqid=0 2024-11-19T15:16:42,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742465_1641 (size=5906) 2024-11-19T15:16:42,722 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T15:16:42,922 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T15:16:42,970 INFO [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/.tmp/ns/bb5f9e01185e4dbdaa78c0cbff75fd80 2024-11-19T15:16:42,987 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/.tmp/rep_barrier/ca76e88b4a3d4e1c9e69c2f264a1dd7e is 102, key is TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480./rep_barrier:/1732029249045/DeleteFamily/seqid=0 2024-11-19T15:16:42,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742466_1642 (size=6025) 2024-11-19T15:16:42,991 INFO [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/.tmp/rep_barrier/ca76e88b4a3d4e1c9e69c2f264a1dd7e 2024-11-19T15:16:43,015 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/.tmp/table/484aba6a99ea45f59fcac681ab834966 is 96, key is TestAcidGuarantees,,1732029222889.9a10ee694987b867f252a8f623c69480./table:/1732029249045/DeleteFamily/seqid=0 2024-11-19T15:16:43,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742467_1643 (size=5936) 2024-11-19T15:16:43,123 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T15:16:43,323 DEBUG [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T15:16:43,419 INFO [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/.tmp/table/484aba6a99ea45f59fcac681ab834966 2024-11-19T15:16:43,422 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/.tmp/info/468f9bec671146938326ccc0f3328000 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/info/468f9bec671146938326ccc0f3328000 2024-11-19T15:16:43,425 INFO [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/info/468f9bec671146938326ccc0f3328000, entries=12, sequenceid=90, filesize=6.4 K 2024-11-19T15:16:43,426 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/.tmp/ns/bb5f9e01185e4dbdaa78c0cbff75fd80 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/ns/bb5f9e01185e4dbdaa78c0cbff75fd80 2024-11-19T15:16:43,428 INFO [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/ns/bb5f9e01185e4dbdaa78c0cbff75fd80, entries=8, sequenceid=90, filesize=5.8 K 2024-11-19T15:16:43,429 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/.tmp/rep_barrier/ca76e88b4a3d4e1c9e69c2f264a1dd7e as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/rep_barrier/ca76e88b4a3d4e1c9e69c2f264a1dd7e 2024-11-19T15:16:43,431 INFO [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/rep_barrier/ca76e88b4a3d4e1c9e69c2f264a1dd7e, entries=6, sequenceid=90, filesize=5.9 K 2024-11-19T15:16:43,432 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/.tmp/table/484aba6a99ea45f59fcac681ab834966 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/table/484aba6a99ea45f59fcac681ab834966 2024-11-19T15:16:43,435 INFO [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/table/484aba6a99ea45f59fcac681ab834966, entries=7, sequenceid=90, filesize=5.8 K 2024-11-19T15:16:43,436 INFO [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1313ms, sequenceid=90, compaction requested=false 2024-11-19T15:16:43,443 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-19T15:16:43,443 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T15:16:43,444 INFO [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T15:16:43,444 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732029402121Running coprocessor pre-close hooks at 1732029402122 (+1 ms)Disabling compacts and flushes for region at 1732029402122Disabling writes for close at 1732029402122Obtaining lock to block concurrent updates at 1732029402122Preparing flush snapshotting stores in 1588230740 at 1732029402122Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732029402122Flushing stores of hbase:meta,,1.1588230740 at 1732029402123 (+1 ms)Flushing 1588230740/info: creating writer at 1732029402123Flushing 1588230740/info: appending metadata at 1732029402144 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732029402144Flushing 1588230740/ns: creating writer at 1732029402551 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732029402566 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732029402566Flushing 1588230740/rep_barrier: creating writer at 1732029402972 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732029402987 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732029402987Flushing 1588230740/table: creating writer at 1732029402994 (+7 ms)Flushing 1588230740/table: appending metadata at 1732029403014 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732029403014Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49a98c7: reopening flushed file at 1732029403422 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21e7e349: reopening flushed file at 1732029403425 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a8357d2: reopening flushed file at 1732029403428 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ef608e2: reopening flushed file at 1732029403431 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1313ms, sequenceid=90, compaction requested=false at 1732029403436 (+5 ms)Writing region close event to WAL at 1732029403440 (+4 ms)Running coprocessor post-close hooks at 1732029403443 (+3 ms)Closed at 1732029403444 (+1 ms) 2024-11-19T15:16:43,444 DEBUG [RS_CLOSE_META-regionserver/50570b5b74e7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T15:16:43,523 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(976): stopping server 50570b5b74e7,46557,1732029218639; all regions closed. 2024-11-19T15:16:43,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741834_1010 (size=25344) 2024-11-19T15:16:43,530 DEBUG [RS:0;50570b5b74e7:46557 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/oldWALs 2024-11-19T15:16:43,530 INFO [RS:0;50570b5b74e7:46557 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 50570b5b74e7%2C46557%2C1732029218639.meta:.meta(num 1732029222422) 2024-11-19T15:16:43,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741832_1008 (size=21697118) 2024-11-19T15:16:43,533 DEBUG [RS:0;50570b5b74e7:46557 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/oldWALs 2024-11-19T15:16:43,533 INFO [RS:0;50570b5b74e7:46557 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 50570b5b74e7%2C46557%2C1732029218639:(num 1732029221402) 2024-11-19T15:16:43,533 DEBUG [RS:0;50570b5b74e7:46557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T15:16:43,533 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T15:16:43,533 INFO [RS:0;50570b5b74e7:46557 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T15:16:43,534 INFO [RS:0;50570b5b74e7:46557 {}] hbase.ChoreService(370): Chore service for: regionserver/50570b5b74e7:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-19T15:16:43,534 INFO [RS:0;50570b5b74e7:46557 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T15:16:43,534 INFO [regionserver/50570b5b74e7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T15:16:43,534 INFO [RS:0;50570b5b74e7:46557 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46557 2024-11-19T15:16:43,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T15:16:43,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/50570b5b74e7,46557,1732029218639 2024-11-19T15:16:43,539 INFO [RS:0;50570b5b74e7:46557 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T15:16:43,540 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [50570b5b74e7,46557,1732029218639] 2024-11-19T15:16:43,541 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/50570b5b74e7,46557,1732029218639 already deleted, retry=false 2024-11-19T15:16:43,541 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 50570b5b74e7,46557,1732029218639 expired; onlineServers=0 2024-11-19T15:16:43,541 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '50570b5b74e7,39501,1732029217890' ***** 2024-11-19T15:16:43,541 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T15:16:43,541 INFO [M:0;50570b5b74e7:39501 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T15:16:43,541 INFO [M:0;50570b5b74e7:39501 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T15:16:43,542 DEBUG [M:0;50570b5b74e7:39501 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T15:16:43,542 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T15:16:43,542 DEBUG [M:0;50570b5b74e7:39501 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T15:16:43,542 DEBUG [master/50570b5b74e7:0:becomeActiveMaster-HFileCleaner.large.0-1732029221144 {}] cleaner.HFileCleaner(306): Exit Thread[master/50570b5b74e7:0:becomeActiveMaster-HFileCleaner.large.0-1732029221144,5,FailOnTimeoutGroup] 2024-11-19T15:16:43,542 DEBUG [master/50570b5b74e7:0:becomeActiveMaster-HFileCleaner.small.0-1732029221145 {}] cleaner.HFileCleaner(306): Exit Thread[master/50570b5b74e7:0:becomeActiveMaster-HFileCleaner.small.0-1732029221145,5,FailOnTimeoutGroup] 2024-11-19T15:16:43,542 INFO [M:0;50570b5b74e7:39501 {}] hbase.ChoreService(370): Chore service for: master/50570b5b74e7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T15:16:43,542 INFO [M:0;50570b5b74e7:39501 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T15:16:43,542 DEBUG [M:0;50570b5b74e7:39501 {}] master.HMaster(1795): Stopping service threads 2024-11-19T15:16:43,542 INFO [M:0;50570b5b74e7:39501 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T15:16:43,542 INFO [M:0;50570b5b74e7:39501 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T15:16:43,543 ERROR [M:0;50570b5b74e7:39501 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-19T15:16:43,543 INFO [M:0;50570b5b74e7:39501 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T15:16:43,543 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T15:16:43,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T15:16:43,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T15:16:43,544 DEBUG [M:0;50570b5b74e7:39501 {}] zookeeper.ZKUtil(347): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T15:16:43,544 WARN [M:0;50570b5b74e7:39501 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T15:16:43,544 INFO [M:0;50570b5b74e7:39501 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/.lastflushedseqids 2024-11-19T15:16:43,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742468_1644 (size=72) 2024-11-19T15:16:43,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T15:16:43,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46557-0x10130ca639e0001, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T15:16:43,642 INFO [RS:0;50570b5b74e7:46557 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T15:16:43,642 INFO [RS:0;50570b5b74e7:46557 {}] regionserver.HRegionServer(1031): Exiting; stopping=50570b5b74e7,46557,1732029218639; zookeeper connection closed. 2024-11-19T15:16:43,643 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@54a5b72d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@54a5b72d 2024-11-19T15:16:43,643 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T15:16:43,954 INFO [M:0;50570b5b74e7:39501 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T15:16:43,954 INFO [M:0;50570b5b74e7:39501 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T15:16:43,954 DEBUG [M:0;50570b5b74e7:39501 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T15:16:43,954 INFO [M:0;50570b5b74e7:39501 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T15:16:43,954 DEBUG [M:0;50570b5b74e7:39501 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T15:16:43,954 DEBUG [M:0;50570b5b74e7:39501 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T15:16:43,954 DEBUG [M:0;50570b5b74e7:39501 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T15:16:43,955 INFO [M:0;50570b5b74e7:39501 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=791.67 KB heapSize=971.44 KB 2024-11-19T15:16:43,970 DEBUG [M:0;50570b5b74e7:39501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/63f6acb9453d4276b83b9ed6451145b8 is 82, key is hbase:meta,,1/info:regioninfo/1732029222507/Put/seqid=0 2024-11-19T15:16:43,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742469_1645 (size=5672) 2024-11-19T15:16:44,373 INFO [M:0;50570b5b74e7:39501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2220 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/63f6acb9453d4276b83b9ed6451145b8 2024-11-19T15:16:44,394 DEBUG [M:0;50570b5b74e7:39501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e5e167a7c3d1451d80d57419e75303c9 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1732029375589/Put/seqid=0 2024-11-19T15:16:44,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742470_1646 (size=46398) 2024-11-19T15:16:44,798 INFO [M:0;50570b5b74e7:39501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=791.12 KB at sequenceid=2220 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e5e167a7c3d1451d80d57419e75303c9 2024-11-19T15:16:44,801 INFO [M:0;50570b5b74e7:39501 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e5e167a7c3d1451d80d57419e75303c9 2024-11-19T15:16:44,816 DEBUG [M:0;50570b5b74e7:39501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/382529e1fe4e4d5ca20971495d577e4d is 69, key is 50570b5b74e7,46557,1732029218639/rs:state/1732029221160/Put/seqid=0 2024-11-19T15:16:44,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073742471_1647 (size=5156) 2024-11-19T15:16:45,220 INFO [M:0;50570b5b74e7:39501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2220 (bloomFilter=true), to=hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/382529e1fe4e4d5ca20971495d577e4d 2024-11-19T15:16:45,223 DEBUG [M:0;50570b5b74e7:39501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/63f6acb9453d4276b83b9ed6451145b8 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/63f6acb9453d4276b83b9ed6451145b8 2024-11-19T15:16:45,226 INFO [M:0;50570b5b74e7:39501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/63f6acb9453d4276b83b9ed6451145b8, entries=8, sequenceid=2220, filesize=5.5 K 2024-11-19T15:16:45,226 DEBUG [M:0;50570b5b74e7:39501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e5e167a7c3d1451d80d57419e75303c9 as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e5e167a7c3d1451d80d57419e75303c9 2024-11-19T15:16:45,229 INFO [M:0;50570b5b74e7:39501 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e5e167a7c3d1451d80d57419e75303c9 2024-11-19T15:16:45,229 INFO [M:0;50570b5b74e7:39501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e5e167a7c3d1451d80d57419e75303c9, entries=176, sequenceid=2220, filesize=45.3 K 2024-11-19T15:16:45,230 DEBUG [M:0;50570b5b74e7:39501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/382529e1fe4e4d5ca20971495d577e4d as hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/382529e1fe4e4d5ca20971495d577e4d 2024-11-19T15:16:45,232 INFO [M:0;50570b5b74e7:39501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36139/user/jenkins/test-data/555b2ee9-5cc5-b50d-bada-c6b4de06b4d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/382529e1fe4e4d5ca20971495d577e4d, entries=1, sequenceid=2220, filesize=5.0 K 2024-11-19T15:16:45,233 INFO [M:0;50570b5b74e7:39501 {}] regionserver.HRegion(3140): Finished flush of dataSize ~791.67 KB/810673, heapSize ~971.14 KB/994448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2220, compaction requested=false 2024-11-19T15:16:45,234 INFO [M:0;50570b5b74e7:39501 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T15:16:45,234 DEBUG [M:0;50570b5b74e7:39501 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732029403954Disabling compacts and flushes for region at 1732029403954Disabling writes for close at 1732029403954Obtaining lock to block concurrent updates at 1732029403955 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732029403955Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=810673, getHeapSize=994688, getOffHeapSize=0, getCellsCount=2421 at 1732029403955Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732029403955Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732029403955Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732029403970 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732029403970Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732029404377 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732029404393 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732029404393Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732029404801 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732029404816 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732029404816Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66d853e7: reopening flushed file at 1732029405223 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7516ccd8: reopening flushed file at 1732029405226 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@546aeaee: reopening flushed file at 1732029405229 (+3 ms)Finished flush of dataSize ~791.67 KB/810673, heapSize ~971.14 KB/994448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2220, compaction requested=false at 1732029405233 (+4 ms)Writing region close event to WAL at 1732029405234 (+1 ms)Closed at 1732029405234 2024-11-19T15:16:45,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42567 is added to blk_1073741830_1006 (size=956196) 2024-11-19T15:16:45,237 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T15:16:45,237 INFO [M:0;50570b5b74e7:39501 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T15:16:45,237 INFO [M:0;50570b5b74e7:39501 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39501 2024-11-19T15:16:45,237 INFO [M:0;50570b5b74e7:39501 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T15:16:45,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T15:16:45,341 INFO [M:0;50570b5b74e7:39501 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T15:16:45,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39501-0x10130ca639e0000, quorum=127.0.0.1:60923, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T15:16:45,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T15:16:45,348 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T15:16:45,348 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T15:16:45,348 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T15:16:45,349 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/hadoop.log.dir/,STOPPED} 2024-11-19T15:16:45,352 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T15:16:45,352 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T15:16:45,352 WARN [BP-618166336-172.17.0.2-1732029215069 heartbeating to localhost/127.0.0.1:36139 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T15:16:45,352 WARN [BP-618166336-172.17.0.2-1732029215069 heartbeating to localhost/127.0.0.1:36139 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-618166336-172.17.0.2-1732029215069 (Datanode Uuid 99eab563-9309-42c7-bee5-e233e87de811) service to localhost/127.0.0.1:36139 2024-11-19T15:16:45,354 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/cluster_26e20745-1875-184e-c7e5-36298de089d9/data/data1/current/BP-618166336-172.17.0.2-1732029215069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T15:16:45,354 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/cluster_26e20745-1875-184e-c7e5-36298de089d9/data/data2/current/BP-618166336-172.17.0.2-1732029215069 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T15:16:45,355 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T15:16:45,363 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T15:16:45,364 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T15:16:45,364 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T15:16:45,364 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T15:16:45,364 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49ca5804-6e8e-2cd2-792b-00a3dd98a8c8/hadoop.log.dir/,STOPPED} 2024-11-19T15:16:45,380 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T15:16:45,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down